J-j_4UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZmZddlmZmZddlmZmZmZmZmZddlmZddlmZdd lmZm Z erdd l!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*ddl+Z+ddl,Z,ddl-Z,ddl.m/cm0Z1ddl2m3Z3m4Z4ddl5m6Z6ddl7m8Z8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@ddlAmBZBddlCmDZDmEZEmFZFddlGmHZHddlImJZJmKZKmLZLmMZMmZmNZNddlOmPZPddlQmRZRmSZSmTZTddlUmVZVmWZWddlMmXZXmYZYmZZZm[Z[ddl\m]Z]m^Z^ddl_m`Z`ddlmaZambZbmcZcmdZdmeZemfZfddlgmhZhdd limjZjmkZkdd!llmmZmmnZndd"lompZpmqZqdd#lrmsZsdd$l/mtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZdd%lmZejeZe,jjed&Ze,jjed'Ze,jjed(Ze,jjed)Zed*Zd+ed,<ed-Zed.Zej,Gd/d0Zej,Gd1d2ZGd3d4Zej,Gd5d6Zej,Gd7d8eZGd9d*Zej:dld:Zdmd;Zdnd<Zdod=Zej,d>?Gd@dAZdpdBZGdCdDZ dqdEZGdFdGeZGdHdIeZGdJdKeZ drdLZ dsdNZGdOdPeZGdQdReZGdSdTeZGdUdVeZGdWdXeZ dt dudYZ dvdZZdwd[Z dxd\Z dyd]Zej,Gd^d_ZejlZdzd`Zd{daZ d|dbZd}dcZd}ddZd~deZd~dfZd~dgZd~dhZGdidMZGdjdkZy)) annotationsN)Counter defaultdict) as_completedFuture)AnyGeneric TYPE_CHECKING TypeAliasTypeVar) ParamSpec OrderedSet)ComputedBuffer Pointwise)CallableIteratorSequence) ModuleType)EnterCudaStreamContextLine)PythonWrapperCodegen)counters dynamo_timed)use_pipelined_autotuning) LambdaFuture PyCodeCache)TritonTemplateCallerBase)get_metric_tableis_metric_table_enabled)get_stream_name) free_symbols)free_symbol_is_typesymbol_is_typeSymT) has_triton)commsconfig config_comms dependenciesirmetrics)can_codegen_without_upcasts)BackendFeatureget_scheduling_for_deviceKernel) estimate_nccl_collective_runtime/estimate_nccl_collective_runtime_nccl_estimator)Dep MemoryDepStarDepWeakDep)GPUTooOldForTriton TritonMissing)count_flops_fx)assign_origin_nodeget_device_typeGraphPartitionSignature MultiOutputMultiOutputLayout NoneLayout)LoopBody)MemoryPlanningInfoForBufferMemoryPlanningInfoForNode)DeviceProperties ReductionHint) green_textred_text)SimplifyIndexing)&_unstable_customized_partition_wrapper cache_on_selfcmpdevice_need_guardget_current_backendget_device_tflopsget_dtype_sizeget_gpu_dram_gbps get_op_namesGraphPartitionMapIndentedBuffer is_collectiveis_cudagraph_unsafe_opis_gpuis_multi_outputs_template#is_output_of_multi_outputs_templateis_wait sympy_product)Vfusion loop_orderingcompute_dependencies cudagraphsBaseSchedulerNoder PartitionType_T_PcleZdZUdZded<dZded<dZded<dZed d Z e d d d Z y) FusionResultNz bool | None should_fusezCallable[[], bool] | None callable_fnLambdaFuture | NonefuturecL|jdu|jduz sJdy)NzLFusion result should contain either fusion decision or callable_fn, not both)rerfselfs g/Users/jameslopez/projects/TradingBot25/.venv/lib/python3.12/site-packages/torch/_inductor/scheduler.py __post_init__zFusionResult.__post_init__ss0  ,1A1A1MN Z Nct|S)N)rerd)clsres rlfusezFusionResult.fusexs  44rnct||S)Nrfrhrp)rqrfrhs rl from_callablezFusionResult.from_callable|s FCCrn)reboolN)rfCallable[[], bool]rhrg) __name__ __module__ __qualname__re__annotations__rfrhrm classmethodrrrurnrlrdrdmsf#K#-1K*1"&F & 55LPD,D6IDDrnrdcBeZdZUded<ded<ded<dZded<d d Zy) PendingFusionrxrfr_node1node2Nrgrhc2|j|jfSrwrrrjs rlget_fusion_nodeszPendingFusion.get_fusion_nodess DJJ''rn)return+tuple[BaseSchedulerNode, BaseSchedulerNode])ryrzr{r|rhrr~rnrlrrs$##  "&F &(rnrc$eZdZdZeddZeddZe ddZeddZ e ddZ e ddZ eddZ e dd Z edd Ze dd Zedd Zedd Zy)MixOrderReductionz This class contains utility functions to decide if we should fuse reductions reducing across different dimensions of the same input tensor. cf|jxr td|jDS)Nc3K|]V}t|trD|jr4t|jtr|jj duXywrw) isinstance SchedulerNode is_reductionnoder _split_size.0subnodes rl z7MixOrderReduction.is_split_reduction..sK+ +'=1$$&7<<8 LL $ $D 0+sAA)rall get_nodesrs rlis_split_reductionz$MixOrderReduction.is_split_reductions3  " s+ >>++ (  rnc`|j|rd}d}|jD]n}t|tr*|j rt|j t s?|j jJtjjjt|j j}|j jJtjjjt|j j}||}|}tjjj||s J|d|tjjj||reJ|d||J||fS|jdS)N v.s. r)rrrrrrr_original_rangesrZgraphsizevarssimplifyrY_original_reduction_rangesstatically_known_equalsgroup)rqrxnumelrnumelr curxnumel currnumels rlget_numel_rnumelz"MixOrderReduction.get_numel_rnumels  ! !$ 'FF>>+w 6,,."7<<@||44@@@GG,,55!',,"?"?@ ||>>JJJGG,,55!',,"I"IJ >&F&F77++CC 4  {3477++CC 4  {341,8% %%F# #::a= rnc|j|}|j|}t|dk7st|dk7s||k(ryt|tt|k(S)NF)rlentuplereversed)rqrrg1g2s rlhas_mix_reduction_ordersz*MixOrderReduction.has_mix_reduction_orderssZ ! !% (  ! !% ( r7a<3r7a<28RyE(2,///rncRd}|jjD]&}t|ts|j|k(s$|}n|sy|j }|jj }|sDt|tsJt||jdjj }|sJt|t|jz sytjjjt!|j"t!|j%ryy)z@ The access to 'buf' is not a broadcast access. NFrT) read_writesreadsrr4nameindex var_rangesFusedSchedulerNodetypesnodesrr"rZrrrrYsizevalues)rqbufr found_depdeprrs rl_is_full_accessz!MixOrderReduction._is_full_accesss  ##))C#y)chh#o * %%00 d$67 HDJ< H7Q33>>Jz:&E4F4F)GG 77   3 3 ).. )=9J9J9L+M rncg}|j|jz}|D]9}|j||s|j||s)|j|;|Srw)used_buffer_namesrappend)rqrrout common_readsrs rlget_common_readz!MixOrderReduction.get_common_reads`..053J3J3LL C""3.33F3FsE3R 3  rnc>t|j||dkDSNr)rrrqrrs rlhas_common_readz!MixOrderReduction.has_common_reads!3&&ue4599rnc|j|}tjjj |d|dzdS)Nrrfallback)rrZrroptimization_hint)rqrrs rl get_numelzMixOrderReduction.get_numels>  ! !$ 'ww11"Q%"Q%-!1LLrnc$|j|Srw)rrs rlget_fusion_scorez"MixOrderReduction.get_fusion_score s }}U##rncxtjjsytjj ry|j r|j sy|jj}|dvst|dk7ry|jr|jsy|j|jzs|j|jzry|j||sytj||}t!|dk(ry|j#|r||}}n|j#|r||}}ny|j%|}|\}} tjj&sd} tjj(j+t-j.|| z| sytjj(j+t-j.|| dzsytjj(j+t-j.|dsyt1d|j3Drytjj(j5| d sytj7|ryt9d |j3D} | S) zP Check whether we can fuse two reductions with mix loop orders. F)cudaxputritonriPric3K|]T}|jrB|jjjtj tj fvVywrw)rrdatareduction_hintrDINNERDEFAULTrs rlrz-MixOrderReduction.can_fuse..[sT 7##% LL   , ,##%%  7sAAi@c3tK|]0}|jr|jjdv2yw)>sumprodN)rrget_reduction_typers rlrz-MixOrderReduction.can_fuse..qs? 2##% LL + + -  2s68)r(rmix_order_reductionrZr cpp_wrapperrU get_devicerrLr ancestorsget_operation_namesrrrris_contiguous_noder#mix_order_reduction_non_strict_moder guard_or_truesympyGeanyrstatically_known_leqrr) rqrr device_typercontiguous_node other_nodernrowncol size_thresrs rlcan_fusezMixOrderReduction.can_fusesJ }}00 77  ||~U\\^&&(--  .";/8;!!#5+=+=+? OOe779 9 OOe779 9++E59)88F |  !  ! !% (*/ZO  # #E **/ZO  ! !/ 2 d}}@@#J 77##11%((4$; 2ST 77##11%((42JK 77##11%((42FG  +446    ww44T9E  / / @ &//1    rnc&|j||Srw)rrs rlare_mix_order_reductionsz*MixOrderReduction.are_mix_order_reductions|s||E5))rnc\tfdjjDsyy)Nc3VK|] }j|j"ywrw)is_contiguous_loadr)rrrqrs rlrz7MixOrderReduction.is_contiguous_node..s' >TsC " "388T 2>T&)FT)rrr)rqrs``rlrz$MixOrderReduction.is_contiguous_nodes, >B>N>N>T>T  rncddlm}|jD]}t|tsJ|j }|j |j}|Dcgc]}|j|k(s|j }}t|dk(rr|D]u} |j| } |j} t| j} tj j"j%| | | } | ddk(rk| ddk(rtyycc}w)Nr)MemoryUsageTyperFT)torch._inductor.loop_bodyrrrr_body memory_usageLOAD buffer_name index_namerindexing_exprsrlistkeysrZrr stride_vars)rqr parent_noderr loop_bodyentriese index_namesr index_exprr var_symbolsrs rlrz$MixOrderReduction.is_contiguous_loads=))+DdM2 22 I,,_-A-ABG18QAAMMS?D6DII,<,<+=>?       v[9I9I9K1L0MN O       v]74;M;M;O3P2QR S tzz?a    vYtzzl; <!!##   vZ0 1q! JJD$$vQZ0'"   S !!!## "!s &F//F8c6|jjSrwrrrjs rlrzSchedulerBuffer.get_nameyy!!##rnc|jJ|jjsy|jjsL|jjs2t |jj t jr4tjjj|jyttjdr|jtjjvrtjj|j}||j j"vr$|j j"|j}n#|j j$|j}tjjj'||jytjjj|jy)Nargs)rshould_allocateget_inputs_that_alias_outputget_mutation_namesrget_output_specr+CommBufferLayoutrZr wrapper_codecodegen_allocationhasattrkernelrinplace_update_buffersrname_to_donated_buffer name_to_bufcodegen_inplace_reuse)rkinput_buffer_name input_buffers rlallocatezSchedulerBuffer.allocatesVyy$$$yy((*  II 2 2 4yy++-$))335r7J7JK GG 3 3DII >  AHHf % 188#B#BB!" ? ?  P  DNN$I$II#~~DD% $ $~~99:KLQQ GG 6 6   GG 3 3DII >rnc|jJt|jjtjst |jry|j D]}t|jtsyyNFT)rrr'r+r?rVr OutputNode)rkuses rlcan_freezSchedulerBuffer.can_freesbyy$$$ dii&& 6:S II; ::C#((J/rnc,i}|D]o}t|j|vr>|j|t|j|t|j<X||t|j<qt|j |_yrw)idrmergerrr)rkrr-rHs rl set_userszSchedulerBuffer.set_usersso&(C#((|v%'*yy388 1E'Fr#((|$'*r#((|$  &--/* rncR|jJ|jjSrw)rr6rjs rlr(zSchedulerBuffer.get_aliases s%yy$$$yy5577rncR|jJ|jjSrw)rr7rjs rlr*zSchedulerBuffer.get_mutations%yy$$$yy++--rncR|jjjSrw)rr8rrjs rlrzSchedulerBuffer.get_devicesyy((*5577rnNrr rr rNonerrv)rrrrUrz Sequence[str]rtorch.device | None)ryrzr{r| dataclassesfieldrrrArrr!r/rrDrIrMr(r*rr~rnrlrrsz O))-K--dCE>C.?k.?.?3/J+ $$($?B +8.8rnrceZdZUdZded<y)SchedulerDonatedBufferNrr)ryrzr{rr|r~rnrlr]r]s ,0K)0rnr]ceZdZUded<ded<ded<ded<ded<ded <ded <d ed <d ed<dZded<ded<ded<dZded<ded<ded<dZded<dTdZdUd ZdVd!Z dVd"Z dVd#Z dWd$Z dVd%Z dXd&Z dYd'ZdZd(Zd[d)Zd\d*Zd]d+Z d^d,ZdXd-Zd_d.Zd_d/ZdXd0ZdXd1Z d`d2ZdVd3ZdVd4Zed_d5Zed_d6Zed\d7Z ed\d8Z!dad9Z"dbd:Z#dcd;Z$ddd<Z%d\d=Z&d\d>Z'd\d?Z(d\d@Z)d\dAZ*d\dBZ+d\dCZ,d\dDZ-dedEZ.d\dFZ/dXdGZ0 df dgdHZ1edhdIZ2edhdJZ3edhdKZ4 didLZ5 djdMZ6edkdNZ7dldOZ8edldPZ9dmdQZ:dndRZ;e< dodSZ=y)pr_OrderedSet[str]rz7tuple[torch.device, tuple[tuple[sympy.Expr, ...], ...]]r last_usager min_input_distancemax_input_distance min_order max_orderrBmpi_nodedict[str, str]mutation_renamesNir.Operation | Nonerlist[SchedulerBuffer]outputsdict[str, SchedulerBuffer]outputs_by_name float | Noneoverride_estimated_runtimedependencies.ReadWritesrOrderedSet[Dep]unmet_dependenciesFrvwrittenc"||_d|_y)NcgSrwr~)r4kwargss rlz,BaseSchedulerNode.__init__..5sBrn)rdebug_device_str)rkrs rl__init__zBaseSchedulerNode.__init__2s$- & rncv||_t|_d|_d|_tt |_d|_|jDcgc]}t|j||c}|_ |jDcic]}|j|c}|_ i|_ycc}wcc}w)NrF)rrr)rrrrarbr r`rr get_outputsrrrjrrlrg)rkroutputrs rl_init_from_nodez!BaseSchedulerNode._init_from_node8s #"#"#$    **,  - ..   -   @D||L|  3|L!#  Ms B1 B6cTt|jd|jdS)Nz(name=)rryrrjs rl__repr__zBaseSchedulerNode.__repr__Rs't*%%&fT]]_,?qAArnc|j}t}|j|dt|jdtt |ddjd|dt |jjd|dt |jd|d t |jj|jz d|d |jd|d |jd|d |j5|jD]!}|j|j# ddd|j!d  |j|j#|j+j-S#1swYXxYw#t$$rt&j)ddYLwxYw)#Longer form printout for trace logsr#(rN) .writes =  .unmet_dependencies = .met_dependencies = .min_input_distance = .max_input_distance = z.outputs = [ r%Ignoring error in debug_str()Texc_info)rrRsplicerrygetattrr)rwritesrqrrarbr+rzr/r&debug_str_extra Exceptionlogwarningr,rstrip)rkrrrs rlr/zBaseSchedulerNode.debug_strUs}} bd  QtGD&$$?@IIJKj))00123WT%<%<=>?74#3#3#9#9D"> ?j33$$&'%4H EOO$6$6$@$@ Aj33..0*2O2O2QR4H z""rncptjd||j|jjy)Nz(%s: unmet_dependencies = %s, writes = %s)rinforqrrrjs rl log_detailszBaseSchedulerNode.log_detailss,  6   # #    # #  rncyNFr~)rkself_dep other_deps rlreorder_loops_by_dep_pairz+BaseSchedulerNode.reorder_loops_by_dep_pairrncd|jjDDcic] }||vr|||c}|_|j|jj |jycc}w)Nc34K|]}|jywrwrrrs rlrz9BaseSchedulerNode.update_mutated_names..sQ-Pc-P)rreads_and_writesrgset_read_writesrename)rkrenamesrs rlupdate_mutated_namesz&BaseSchedulerNode.update_mutated_namessuRT-=-=-N-N-PQ! Qw '$- Q!  T--44T5J5JKL ! sA2cX|j|jj|yrw)rr with_readrkrs rl add_fake_depzBaseSchedulerNode.add_fake_deps! T--77<=rncBtd|jDS)Nc3`K|]&}|jxs|j(ywrw)r(r*rrs rlrz=BaseSchedulerNode.has_aliasing_or_mutation..s- @RCOO  4!2!2!4 4@Rs,.)rrzrjs rlhas_aliasing_or_mutationz*BaseSchedulerNode.has_aliasing_or_mutations% @D@P@P@R   rnch||_|jj|_|jyrw)rrrq prune_deps)rkrws rlrz!BaseSchedulerNode.set_read_writess("&"2"2"8"8 rncb|j}tfd|D}||z |_y)Nc3BK|]}j||ywrw)get)rkmutation_real_names rlrz3BaseSchedulerNode.set_last_usage..s !U 1"4"8"8A"> )used_or_aliased_buffer_namesrr`)rkfuture_used_buffersr used_bufferss ` rlset_last_usagez BaseSchedulerNode.set_last_usages088: !!U !UU &)<<rncF|jD]}|jyrw)rjrD)rkrs rlmark_runzBaseSchedulerNode.mark_runs<.s W HHWr)r itertoolschainrrrrjs rlrz#BaseSchedulerNode.used_buffer_namess?  t'7'7'='=t?O?O?V?VW   rnc\ttj|jj|jj Dcgc]*}t |tr |js |j,}}t|dkDr|j}j|tjjj!|rC|j#fdtjj|j%Dt|dkDrScc}w)z Returns buffer names used by this node, including aliases. Note: is_fake WeakDeps are excluded since they are purely for ordering and should not affect buffer lifetime. rc3*K|] }|vr| ywrwr~)ralias used_namess rlrzABaseSchedulerNode.used_or_aliased_buffer_names..s'"5J. "5s)rrrrrrrr6is_fakerrpopaddrZrname_to_bufferrextendr6)rkrdepsrs @rlrz.BaseSchedulerNode.used_or_aliased_buffer_namess'1l !t'7'7'='=t?O?O?V?VW WsG, HHW  $i!m((*C NN3 ww%%))#. !"!7!7"224"5 $i!m! s /D)cLtfdjD_y)Nc3fK|](}|jjjvr|*ywrw)rravailable_buffer_namesrrrks rlrz/BaseSchedulerNode.prune_deps..s/- .xxt~~DDD .s.1rrqrjs`rlrzBaseSchedulerNode.prune_depss#",- ..- # rncdfd tfdjjD}jjj |y)Nct|tsy|jjjvryjj|jj }|t jjvSr) rr6rrr@rrZrremoved_operations)rop_namerks rl should_prunez7BaseSchedulerNode.prune_weak_deps..should_prunes_c7+xxt~~999nn00:KKMGagg888 8rnc34K|]}|s |ywrwr~rrrs rlrz4BaseSchedulerNode.prune_weak_deps..s 1C\#5FC1 rr3rrv)rrrr remove_reads)rk to_removers` @rlprune_weak_depsz!BaseSchedulerNode.prune_weak_depssN 9 ++11   T--::9EFrncFt|||jjyrw)_prune_redundant_depsrr@)rkname_to_fused_nodes rlprune_redundant_depsz&BaseSchedulerNode.prune_redundant_depss d$68R8RSrncR|jJ|jjSrw)rget_operation_namerjs rlrzBaseSchedulerNode.get_namerPrnc"|jSrwrrjs rlget_first_namez BaseSchedulerNode.get_first_names}}rncBtd|jDS)Nc3<K|]}|jywrwrrrs rlrz8BaseSchedulerNode.get_operation_names..sG6Fd$--/6F)rrrjs rlrz%BaseSchedulerNode.get_operation_namessGdnn6FGGGrnc:td|jDS)Nc3<K|]}|jywrwrrrs rlrz5BaseSchedulerNode.get_buffer_names..sALS#,,.Lr)rrjrjs rlget_buffer_namesz"BaseSchedulerNode.get_buffer_namessADLLAAArncBtd|jDS)Nc3ZK|]#}t|txr t|d%yw)T)disallow_fp32_opsNrrr-rns rlrzABaseSchedulerNode.can_codegen_in_low_precision..s7 & q- ( G+AF G%s)+rrrjs rlcan_codegen_in_low_precisionz.BaseSchedulerNode.can_codegen_in_low_precisions% ^^%   rncBtd|jDS)Nc3VK|]!}t|txr t|#ywrwrrs rlrz@BaseSchedulerNode.can_codegen_without_upcasts..s- % q- ( K-H-K K%s')rrjs rlr-z-BaseSchedulerNode.can_codegen_without_upcastss# ^^%   rnc|gSrwr~rjs rlrzBaseSchedulerNode.get_nodes s v rnc|jSrw)rjrjs rlrzzBaseSchedulerNode.get_outputss ||rnc |j|Srw)rl)rkbuf_names rl get_outputzBaseSchedulerNode.get_outputs##H--rncR|jJ|jjSrw)rrrjs rlrzBaseSchedulerNode.get_devices%yy$$$yy##%%rncL|j}|duxr|jdk(SNcpu)rrrkdevices rlis_cpuzBaseSchedulerNode.is_cpus'"T!:fkkU&::rncX|j}|duxrt|jSrw)rrUrrs rlrUzBaseSchedulerNode.is_gpus'"T!9fV[[&99rncyrr~rjs rlrzBaseSchedulerNode.is_reduction"rncyrr~rjs rlis_native_matmulz"BaseSchedulerNode.is_native_matmul%rrncyrr~rjs rl is_split_scanzBaseSchedulerNode.is_split_scan(rrncyrr~rjs rl is_templatezBaseSchedulerNode.is_template+rrncyrr~rjs rl is_externzBaseSchedulerNode.is_extern.rrncyrr~rjs rl is_foreachzBaseSchedulerNode.is_foreach1rrncyrr~rkread_deps rl can_inplacezBaseSchedulerNode.can_inplace4rrncyrr~rjs rlhas_side_effectsz"BaseSchedulerNode.has_side_effects7rrnc  ddlm}ttrtj rt jjjtjr{tt jtjjj j"rt%t jddt't jdsyj(t jj*zj,j.z}dfd }j1D]Q}|j2}|J|j5rr|j7sb|j9sR|j;t jj<vs(t|j?t@jBrjDjFD]}|jHj,jJvr$j,jJ|jH}n/j,jLjO|jH}|s|t jjPjS|st|jTtVr|jXJ|jXDcgc] }|j2j;|vr|"} }j,j[|jH} | r/t]| dk(s?| dj^sP| dj2usc|j2qt|j2j?t@j`t@jbt@jdt@jBfr|jTrft|jTj2t@jft@jhfr(t]|j2j7dkDrE||j2|j2sd||snt jjjjm|j;|j;tt jtjjj j"rnt jjnjq|j;t jjnjq|j;|j;t jjr|j;<QTycc}w) z~ Decide if there should be inplace updates for the node and record the decision in the active kernel. r)can_match_buffer_size mutationsNr4c|jj}|jt}|jD]}|j }t |ts |j|jjvs|jj||urd|fd|jjDz}t|dkDsyy)Nc3@K|]}|jk(r|ywrwr)rors rlrz^BaseSchedulerNode.decide_inplace_update..single_index_in_fused_node..ms%Evv)ErFT) rget_fused_noderrrrrr_rrrrr)buf_to_be_inplaced fused_noderr. user_noderrks @rlsingle_index_in_fused_nodezKBaseSchedulerNode.decide_inplace_update..single_index_in_fused_nodeUs ,55DDTJJ)224H%/LD*00 II !)->?,,.-77JJK)33BB9M%&&22CCE t9q= '1*rnr)r3rrrv):codegen.wrapperr,rrr(inplace_buffersrZr has_featurerr.INPLACE_BUFFERSr=rrcodegensimd SIMDKernelrr<rrrcompleted_operationsrzrr5r6r7rremoved_buffersr8r+r9rrrr?r@rr: can_reuserNopKernelSchedulerNoderhas_cross_stream_hazardrr(r?r>MutationLayoutSHOULDREMOVEFallbackKernelr=r4 make_inplacer-rr>) rkr,inconsequential_nodesr6rbuf_noderead input_bufxremaining_usesrBs ` rldecide_inplace_updatez'BaseSchedulerNode.decide_inplace_update:s ; t] +&&##DOO$5~7U7UVqxx)@)@)E)E)P)PQ188[$7C&)  NNgg(( )nn11 2  D##%CxxH' '',,.88:..0<<>QWW%<%<<h668":M:MN((..99 E EE $ E Edii PI $ : : > >tyy II,,66y$G&y'<'<>TU$??666"+&!0A66??,4II!0#& /3nn.T.T 4/+ 4/14*1-99*1-22d:%NN6 *%NN::< " " 4 4 " = = " 3 3 !&11 * ) 5 5 : :!#!2!2BNN C!!$INN$O$O$Q RUV V1)..#((K6yA  2293E3E3GX%HHeoo&=&=&B&B&M&MHH..2293E3E3GHHH..223<<>B&..077 G}/&6&s?%V cRtjsy|r |jry|jJ|jj }g}|D]0}|j dk(r|j d|j dd|j d|j}d|jvr|d|jdz}|j |d|jvs|jd}|jd d d }|j d|jd djddjddjddz|j d|j d3t|dk(ry|j|d|_y)Nr{rz#pragma CMT ORIGIN:z #pragma CMT  seq_nrz seq_nr: stack_trace|r)maxsplitr{z{{}z}}r\z\\z#pragma CMT END ORIGINrT) r(comment_originrrr get_originsrrtargetmetarsplitreplacer writelines) rkbuffer only_onceorigins out_linesr0 op_info_strrPstack_trace_last_lines rlcodegen_originating_infoz*BaseSchedulerNode.codegen_originating_infos$$   yy$$$))'') Attx   R   2 3(az:K166!)hqvvh7G6H,II   [ )&!" !6 7 (3(:(:3(:(KB(O%  "+33C>WS$'WT4(Wf   !9:  $36 y>Q   )$ rnc(|jddS)NT include_readsinclude_writes!get_read_write_buffers_sizes_implrjs rlget_read_write_buffers_sizesz.BaseSchedulerNode.get_read_write_buffers_sizess55t6  rnc(|jddS)NTFrerhrjs rlget_read_buffer_sizesz'BaseSchedulerNode.get_read_buffer_sizess55u6  rnc(|jddS)NFTrerhrjs rlget_write_buffer_sizesz(BaseSchedulerNode.get_write_buffer_sizess556  rncZt|j||jdS)Nrer)start)rget_read_write_buffer_accessesr)rkrfrgs rlriz3BaseSchedulerNode.get_read_write_buffers_sizes_impls3  / /+N 0 fh   rnc ttriSttrtjtriSttr`tjt j r<jjtjjjuriSd dttr@tjdtjdzn tdt!j"t$}|r9j&j(D] }||j*j-|"|r9j&j.D] }||j*j-|"|r&t1dj&j(Dn t1}|r&t1dj&j.Dn t1}d fd tt2rt1fd|D}||z }||z }i}||zD]} t5fd || D | t6j8j:vrt6j8j:| } n;| t6j8j<vrt6j8j<| } n d fd  | } | |vr| || <|| xx| z cc<|S)az Counting the number of bytes accessed for a kernel is surprisingly tricky. In particular, there is a differentiation between 'theoretical' memory accesses and practical memory accesses. For example, a layernorm kernel may actually access an input 3 times, but in theory, it only needs to access its input once (and may be optimized to do so through say, persistent reductions) Another example is that even though a buffer is passed in, we may not access the entire buffer. This may occur if we are accessing a slice of the buffer. Another tricky case is for indirect indexing, where the amount of bytes accessed depends on the values of the input. What this function aims to compute is the memory accesses for worst-case inputs, best-case optimization. What this means is that for each buffer we compute the amount of potential accesses in two ways and take the minimum. 1. Numel in ranges multiplied by number of deps the buffer has 2. The buffer size Returns memory accesses per buffer. cXtjjj|dS)Nrr)rZrrr)ss rl try_size_hintzGBaseSchedulerNode.get_read_write_buffer_accesses..try_size_hint:s"77##55a!5D DrnrreAc34K|]}|jywrwrrs rlrzCBaseSchedulerNode.get_read_write_buffer_accesses..OsB+ACsxx+Arc34K|]}|jywrwrrs rlrzCBaseSchedulerNode.get_read_write_buffer_accesses..TC+BCsxx+Brcjj|j}td|D}t |t|z dkDS)Nc34K|]}|jywrwr)rr.s rlrz\BaseSchedulerNode.get_read_write_buffer_accesses..is_materialized..[s!>$))rr)rr@rrr)rrrbuf_usesrks rlis_materializedzIBaseSchedulerNode.get_read_write_buffer_accesses..is_materializedYsGNN..s399E!!>!>>Hx*V"4459 9rnc3JK|]}|jr|ywrwr)rrr}rks rlrzCBaseSchedulerNode.get_read_write_buffer_accesses.._s")%_S$++-Nvs##c3"K|]}ywrwr~)rr node_numels rlrzCBaseSchedulerNode.get_read_write_buffer_accesses..hs$R;QCZ;Qs cB|syt|tjr|jSt|jt r͉j j|jj}d}|D]}t|jtrt|jtsJt|jjtr5|jjD]}||jz }y|St|jtjr"t!fd|j#DS t%|j'}t)|j+t-|zS)Nrc3hK|])}tjj|+ywrw)rZr get_buffer)rmut_name get_buf_bytess rlrzZBaseSchedulerNode.get_read_write_buffer_accesses..get_buf_bytes..s-(@H&agg&8&8&BC(@s/2)rr+TorchBindObjectrr'r>rr@rrrrGr_r=rzr?rr7rYrrN get_dtypemin) rrtotr. sched_buf buf_elemsbuf_accessed_elemsrrkrus rlrzGBaseSchedulerNode.get_read_write_buffer_accesses..get_buf_bytesqsIc2#5#56,,.. ,=>!NN66s||~FLLEC %%dii<$)$))5FGGG%diinnkB-1YY-B-B-D #}Y^^'D D.E $%!&J BMM:(+(>(>(@ !.mCLLN.K LI)#--/:S*I>rn)rtz sympy.Exprrr )rr rSequence[BaseSchedulerNode]rrv)rz4ir.Buffer | ir.TensorBox | ir.TorchBindObject | Nonerr )rrAExternKernelSchedulerNoderr=r+rD op_overloadr_prims rng_primsgraphsafe_run_with_rng_staterrY get_rangesr  collectionsrrrrrrrrrrrZrr graph_inputs)rkrfrg buf_accessesrrrr?buf_byte_accessesrr buf_bytesrrr}rrus` @@@@@rlrqz0BaseSchedulerNode.get_read_write_buffer_accessess6 d2 3I d5 6: II{< I t6 7499b&7&78 %%||%%BBCI E dM *&doo/23 1! 456J SJ"..t4 ''--SXX&--c2. ''..SXX&--c2/  B4+;+;+A+AB B  C4+;+;+B+BC C   : d. /()%)Oo-FO+E,.H!$$R<;Q$R!R 177111gg,,X6QWW111gg**84# I# # # J&c*I00.7!(+!(+y8+g'j! rncT|jy|jj}|yt|}|yt|tj r|jj }tjjj|d}tddxx|z cc<|S)Nrrinductor flop_count) rget_origin_noder9rrSymIntexprrZrrrr)rkfx_nodeflopsresolved_flopss rlestimate_flopsz BaseSchedulerNode.estimate_flopss 99 ))++- ?w' = eU\\ *JJOOE));;EA;N\*n<*rncR|j |jS|jSrw)rn_get_estimated_runtimerjs rlget_estimated_runtimez'BaseSchedulerNode.get_estimated_runtimes)  * * 622 2**,,rnc|jdjd}|jj}t t |syt |jrt|jtjsJ tjrst|}t}|j|}|t|tsJ|St!|}|t#|j}|j%|||St#|jSt/|jryt1|}||S|jj3} t5} t7| dz} | dkrt9d| | dkrt9d|  |j=} | dk(s| |j?| z } | dz }|Sd}|j?}|dn|}|| z| z d z}|| z }tA||} | dz }|S#t&$r}t(j+|Yd}~yd}~wt,$r}t(j+|Yd}~yd}~wwxYw#t:$rYywxYw) zC Returns estimated op runtime in milliseconds (ms) rNvaluelJ)z-gpu_memory_bandwidth cannot be <= 0, but got z"gpu_flops cannot be <= 0, but got g.Ag?rv)!rrzrr8rUr;rSrr+IRNoder),runtime_estimations_use_nccl_lib_estimations)get_estimate_runtime_cache_key_from_snodeget_estimate_runtime_cachelookupfloatr2r1 set_value ValueErrorrr TypeErrorrX maybe_estimate_runtime_benchmarkmaybe_get_dtyperOrMAssertionErrorrrrjmax)rkrr' cache_keycache cache_valmsrretdtypegpu_memory_bandwidth gpu_flops flops_estnsfactor counted_bytes compute_time transfer_times rlrz(BaseSchedulerNode._get_estimated_runtimesw nnq!--/2))+of-.  #dii3 33 LL I$ OI68E % Y 7I ,))U;;;((HNBz=diiHOOIRO8I7 BBTYY  .t4 ?J((* #4#6 )%069I$q($CDXCYZA~$'I)%UVV '') >Y.2247KKBcBI99; *2 *Y6#= %(<< } - #X o     :  sCA H6HH(>I$ I!H66 I!II!$ I0/I0cyrwr~rjs rlget_template_nodez#BaseSchedulerNode.get_template_nodernc.|j}|J|Srwr)rktemplates rlget_template_node_or_throwz,BaseSchedulerNode.get_template_node_or_throws!))+###rncftdt|D}|d|}||}||dzd}|||fS)zQ For the list of nodes, get the prologue, template, and epilogue c3HK|]\}}|js|ywrwr )rirs rlrzCBaseSchedulerNode.get_prologue_template_epilogue..sP,ryrzr{r|rrnrrrxr|rr/rrrrrrrrrrrrrrrrrrrIrrr r-rrzrrrrUrrrr r"r$r(r*rLrcrjrlrnrirqrrrrrrrr~rnrlr_r_s BB NN''$$ $D $ ""///3 3((''GT #4B*6+ # !.7  M>  =#2=HV= = 6 G T">T T .HHBB    .&;:IX9=-$-15- -^       ! 37  L!!L!37L! L!\$- UUn  1& 1 S 1 1rncRtjjjSrw)rr codecache LocalCacher~rnrlrr&s ?? $ $ / / 11rnct|jdd}|jj}|jjg||jj|jj }|jj }t j||f\}}ddt|ftfd|Dz}|S)Npython_kernel_namerct|tjxr+t|tjtjf Srw)rr+rGeneratorStateOpaqueObjectStaterJs rl _is_tensor_irz@get_estimate_runtime_cache_key_from_snode.._is_tensor_ir5s<!RYY' !!2#7#781 -  rnc3dK|]'}|rt|jnd)ywrw)rr)rars rlrz.<s)U9a}Q'7ajjl#TA9s-0rV) rrinputsfill_non_provided_args constant_argsrupytree tree_flattenr r)snoderr4ru flat_argsflat_args_pytree_specrrs @rlrr+s -A2F ::  D :: , ,*$*))*  DZZ  F'-':':D&>'J$I$   U9U U VI rnct|tsytjjj tjjj tjjjd}t|jdd}||vryt|jtjsy||S)N)zextern_kernels.mmzextern_kernels.bmmzextern_kernels.addmmrr) rrropsatenmmbmmaddmmrrr+ ExternKernel)rmms_fnsrs rl_get_mm_like_fnrAs e6 7"YY^^..#iinn00 %  4 4G !-A2F( ejj"// 2 % &&rncZ d}d}tjrt}|y|} fd}nyt}t }|j |}|t |tsJ|Sddlm |\}}ddl m } | j|||ddd} |j|| | S) NcSrwr~)rsnode_args_kwargssrlrvz2maybe_estimate_runtime_benchmark..Zs !25!9rnr)rr) benchmarker )memory_warmup_itersbenchmark_itersmax_benchmark_durationr)r(!runtime_estimations_mms_benchmarkrrrrrrutilsr$torch._inductor.runtime.benchmarkingr benchmarkr) rbench_fnargs_kwargs_fnmm_fnrrrr4rurrrs ` @rlrrQsHN //& =99%@I & (E Y'I)U+++(!#LD&@    !  B OOIRO( IrnT)slotscNeZdZUded<ded<ded<ded<d dZd dZd d Zy ) WhyNoFuser name1name2reasonztuple[Any, ...]r4cX|j|_|j|_yrw)rrrrkrrs rlrxzWhyNoFuse.__init__~s^^% ^^% rncJ||_||_tj|yrw)rr4 fusion_logdebug)rkrr4s rl__call__zWhyNoFuse.__call__s  rncpd|jd|jd|j|jzzS)Nz cannot fuse z with r#)rrrr4rjs rl__str__zWhyNoFuse.__str__s6djj\ |2> KK$)) #  rnNrr_rr_rrU)rr r4rrrUrR)ryrzr{r|rxrr!r~rnrlrrws& J J K &  rnrct|ttfrt|t}t j |d}d|vrdtj|dS|S)Nkey)r+r ) rrsetsortedr pprintr)textwrapr+)objr-s rlr)r)sR# C()Sc" ^^C *F v~HOOFG4566 Mrnc0eZdZddZddZddZd dZeZy) rGc&t|g|_yrwrrs rlrxzOutputNode.__init__s",cU"3rncyrr~rjs rlrzOutputNode.is_reductionrrncy)Nr~r~rjs rlr6z'OutputNode.get_inputs_that_alias_outputrrncy)NOUTPUTr~rjs rlrzOutputNode.get_namesrnN)rr5rrUrVrWrR)ryrzr{rxrr6rrr~rnrlrGrGs4HrnrGctjjD]N}t|tr|j j }|jxxdz cc<Pdfd tfdjD}|r?j|z _jjj|yy)am Prunes weakdeps intended for mutation ordering on an upstream fused node if after fusion there is another dependency on the fused upstream node, making the weakdep redundant In essence this enforces an ordering on fusions. As fusions occur, weakdeps will be incrementally removed, enabling other fusions, ensuring they are fused in order. rct|trf|jj}|j dkDxr j j ||}|k(}|xs|Sy)NrF)rr6rrrrfusable_weak_dep)rr is_redundant is_self_depr@name_to_dep_countrrs rlrz+_prune_redundant_deps..should_prunes c7 #!#((+<<>G,"7+446nn55'0$ -W5=K.; .rnc34K|]}|s |ywrwr~rs rlrz(_prune_redundant_deps..s.,s2C.rNr) rrrqrr6rrrrrrr)rrr@rr deps_to_pruner8rs``` @@rlrrs'2&9&9&;&&#w'!#((+<<>G 09BBD E J E'  ..M"&"9"9M"I T--::=IJrncHeZdZdfd ZddZd dZd dZd dZd dZxZ S) rcnt|||j||j|j t |t jr[|jrJtj|jdj}d}|j}|||ff|_yyyNrr)superrxr|rget_read_writesrr+UserDefinedTritonKernelcan_fuse_epiloguemathr mutable_argsshapeget_device_or_errorr)rkrrnumelrr __class__s rlrxz"ExternKernelSchedulerNode.__init__s # T" T1134 dB66 7DEGR= Bxrnct|jtjsJ|jj |Srw)rrr+rr;)rkwrappers rlr;z!ExternKernelSchedulerNode.codegens/$))R__555yy  ))rnrrrrrrUrRrVrSequence[Sequence[sympy.Expr]]rNrrrU) ryrzr{rxrr"r*rr; __classcell__rGs@rlrrs# 3cW*rnrc eZdZdfd ZxZS)rAct|||j||j|j yrw)r>rxr|rr?rkrrrGs rlrxzNopKernelSchedulerNode.__init__s5 # T" T1134rnrO)ryrzr{rxrSrTs@rlrArAs 55rnrAceZdZUdZded<ded< d!fd Z d" d#dZ d" d$dZ d%d Zd&d Z d'd Z d(d Z d'd Z d)dZ d'dZ d*dZd+dZd,dZd-dZd-dZd-dZd-dZd.dZd/dZ d0dZd1dZ d2 d3dZed4dZed4dZd5dZed6dZed-fd Z xZ!S)7rzu A SchedulerNode is a node for scheduling that encapsulates either a ComputedBuffer or a TemplateBuffer. z tuple[Sequence[sympy.Expr], ...]_sizesr@rcft|||j||jyrw)r>rxr|_compute_attrsrWs rlrxzSchedulerNode.__init__ s, # T" rnct|jtjtjfsJ|jj ||\|_}||_|jj}|jj|j}|||j f|_ tj xst|j  }t|jtjr,|j#|jj%|y|j#t'j$|jg|j d|iy)Nextra_indexing_constraintsrecompute_sizes_body_func) normalizer`)rrr+rTemplateBuffersimplify_and_reorderrYrrEr get_backendgroup_fnrr(loop_ordering_after_fusionrUrrextract_read_writesr*)rkr^r_bodyrrdshould_normalizes rlr[zSchedulerNode._compute_attrss7 $))b&7&79J9J%KLLL II::'A&?;  T ..0>>--f5>>ht{{34  &@@@  KKI E  dii!2!2 3   --8H-I   00JJ!%8H rnctd|jjD}|j|||rD|j |jj |j |jyy)Nc3NK|]}t|ttfs|ywrwrr6r5rs rlrz8SchedulerNode.recompute_size_and_body..8#0 1CZgwEW5XC1%%r])rrrr[rrrrg)rkr^r_ fake_depss rlrecompute_size_and_bodyz%SchedulerNode.recompute_size_and_body3s &00 ++110 &  'A&?       **95<.Irlrmr`rSIMDScheduling)rrrrr*rfrrYrrrgpointwise_read_writes clear_cache codegen.simdrscandidate_tilings cache_clear)rkr`need_clear_tiling_cachernrss rlrefresh_dependenciesz"SchedulerNode.refresh_dependenciesDs &00 ++110 &   , ,  ![[ 4= Yy ! VD)) *   ""..t4 " 4  , , 8 8 : #rnc|jj||_|jj|_|j ddy)NFTr`ry)rreorder_iter_loopssizesrYrz)rk new_orders rlapply_new_loop_orderz"SchedulerNode.apply_new_loop_orderbsAZZ22   jj&&  !!E4!Prnc|jj}t|jj|z }t t |}t t |||z}|j ||zt|jddk(sJ|jd|jdd|jddff|_y)Nrrr)rget_original_num_rdimsr iter_varsrrangerr)rk num_rdims num_pwdimspwdimsrdimss rlswap_pw_red_dimensionz#SchedulerNode.swap_pw_red_dimensionjsJJ557 --.: uZ()eJ Y(>?@ !!%&.14::a=!Q&&&ZZ]TZZ]1%5tzz!}Q7G$HH rncD|jj|_|Srw)rextract_pw_from_reductionrjs rlrz'SchedulerNode.extract_pw_from_reductiontsZZ99;  rnctj|syt|jtj sJ|jj 5|jdddy#1swYyxYwrw)rrrrr+rwith_original_inner_fnr[rjs rlcancel_reduction_splitz$SchedulerNode.cancel_reduction_splitxsU 33D9 $))R%6%6777 YY - - /    !0 / /s A11A:ct|jtjtjfsJ|j j |||_|j j|_|jj}|jj|j}|||jf|_ |jddy)NTr|)rrr+rrar#expand_dimension_for_pointwise_noder~rYrErrcrdrrz)rk dimension new_rangerrds rlrz1SchedulerNode.expand_dimension_for_pointwise_nodes$))b&7&79J9J%KLLLZZCC y  jj&& ..0>>--f5>>ht{{34  !!D$!Ornc|jj|_|jj|_|j ddy)NTFr|)r merge_loopsr~rYrzrjs rlrzSchedulerNode.merge_loopss<ZZ++- jj&&  !!D%!Prncd}|jd}t||jcxk(r|jk(rnn|j|}|rPtxj dz c_t jd|j||j|yt jd|jy)Nrrz"Reorder loops for %s with order %sTzEDon't reordering %s because we can not decide the suitable loop orderF) rYrnum_varsdecide_loop_order_to_matchr,num_loop_reorderingloop_ordering_logrrr)rkrrr self_sizess rlrz'SchedulerNode.reorder_loops_by_dep_pairs [[^ z?h// E93E3E E ;;IFI   ' '1 , '  # #4dmmoy   % %i 0  # #W  rnc$|j}|d|jd|d|jd|d|jg}|jj D]}t |t r|j}tjj|}t |tjrZ|j|dt|jt |j t"rR|jd|d|jt%j&|j j)d |j*J|j-|j/d j1|S) Nz.group.device = rz.group.iteration = rz .sizes = z _layout = zclass z _loop_body:r'r)rrrYrrrr6rrZrrr+rrr)r'rr@r+r+r/rrrjoin)rkrlinesrrrs rlrzSchedulerNode.debug_str_extrasD}}f$TZZ]O 4f' 1  7fIdkk] +  ##446Cc7+88gg((2!#r'9'9:LLH:Z 8K7L!MN 7 djj( + LL6${3 4 LL)=)=)?H Iyy$$$ T//12yyrnc|jSrw)rYrjs rlrzSchedulerNode.get_ranges {{rnc<t|jtjtjfsJdt |jt |jjxr'|jduxs|jj SNtype(self.node)=) rrr+rrarrvrrhas_partial_accumulaterjs rlrzSchedulerNode.is_reductions$))b&7&79J9J%KL tDII ! LDII0023 JJ$  Gdjj&G&G"G rnct|jtjsJdt |j|jj dk(S)Nrdot)rrr+rrrrjs rlrzSchedulerNode.is_native_matmulsJ$))R%6%67N.0s?,>Ss ,>rrztype(write_dep)=)r rrzrrrrr*r4riterrrr)rkr' write_deps rlr(zSchedulerNode.can_inplace-s     ?D,<,<,>? ? t&& '1 , l,,2 T$"2"2"9"9:;Ii)?)?@ WEUT)_DVBW W@>>Y__4X)..9X Xrnct}t|jtr|jj D]}|j dk(s|j dk(s#d|jvr|jddk(s,t|jdk(s\|jddk(so|jd|jvr|jdn(t|jdk\r|jd nd |S) N call_methodstoremode atomic_addrr&rrrr) rrrr@rrrXrurr4r)rkbuffers_store_as_atomic_addrs rl_get_atomic_add_buffersz%SchedulerNode._get_atomic_add_buffers:s7A|# djj( + ,,.GG}, w.4;;.4;;v3F,3V Na/DIIaLL4P033!T[[0 F+.1$))n.Adiilr/+*rncp|j|jjdryt| S)Ndevice_assert_asyncT)rhas_opr>r*rkrGs rlr*zSchedulerNode.has_side_effectsNs2 :: !djj&7&78M&Nw'))rn)rrrz%ir.ComputedBuffer | ir.TemplateBufferrrUNN)r^'tuple[dict[Any, Any], list[Any]] | Noner_zCallable[_P, _T] | NonerrU)r^rr_zCallable[..., Any] | NonerrU)r`rvryrvrrU)r Sequence[int]rrUrTrr_)rr rr rrUrrRrPrVr)rSequence[sympy.Expr]rrU)rrQrzdict[sympy.Expr, sympy.Expr])rrQrrUr)rrvrro)rrorr)"ryrzr{rr|rxr[rorzrrrrrrrrrrrrr rrrr;rrIrtrr(rr*rSrTs@rlrrs -, O4  OS=A$K$;  FOS?C$K$=  ";;8<; ;<QI"PP),P P" Q!.7 . ,  7 8O! 8  % 0!%      GG HH ++&**rnrc nj}jtjj |Dcgc]}|j c}t fdt j|Dcgc]}|jc}Dj jz _ycc}wcc}w)Nc3ZK|]"}|jjvr|$ywrwrr)rr group_snodes rlrz2refresh_group_node_dependencies.._s. Pxx{;;== P(+) rrr* ReadWrites merge_listrrunionrqr)rrrJs` rlrefresh_group_node_dependenciesrVs  F**6+J6aAMM6+JK  !'')O1!*>*>)OP   ! ! ( (  )",K *Ps B- 0B2rclt|ttfsJ||_||_d|_t j|Dcgc]}|j|jc}|_t|td|jD|_ td|jD|_ td|jD|_td|jD|_|j!Dcic]}|j#|c}|_ycc}wcc}w)Nc34K|]}|jywrwrcrrJs rlrz"init_group_node..wH5G 5Grc34K|]}|jywrw)rdrs rlrz"init_group_node..xrrc34K|]}|jywrw)rars rlrz"init_group_node..y)&8&8rc34K|]}|jywrw)rbrs rlrz"init_group_node..|rr)rrGroupedSchedulerNoderrrrrrrrrcrrdrarbrzrrl)rrrrJrs rlinit_group_noderhs! k$68L#M NN NK%KK&,,% Av!)@!++v AK$K0H[5G5GHHKH[5G5GHHK%()&1&8&8)&K"&))&1&8&8)&K"(3'>'>'@#'@ '@#K B#sD,D, D1ceZdZUdZded<e d!dZd"dZd#dZe d$dZ d%dZ d&fd Z e d'd Z d'd Ze d(d Zd)d Zd'dZd'dZ d*fd Ze d(dZe d(dZd+dZd'dZe d,dZe d,dZe d,dZe d,dZe d-dZd.dZe d,dZd/dZd0dZ d1dZ!d'dZ"e d,fd Z#xZ$S)2rz This is a "fake" scheduler node that represents a group of scheduler nodes that are meant to be fused together. The way it does this is by maintaining its unmet dependencies as the union of its constituent nodes. rrc|j|jusJt|ttfsJ|j r6t|t r&t|j tjsJt|ttfsJttj|j|j}||j|Srw) rrrrr rrr+r=rrrr)rqrrrs rlrrzFusedSchedulerNode.fuses%//111%-1C!DEEE    :e5N#Oejj"..9 99em5G%HI IIY__U__%68IJK5??E**rnc|jD]6}t|tsJ|jsJ|j 8|Srw)rrrrrrkrs rlrz,FusedSchedulerNode.extract_pw_from_reductionsE{{Gg}5 55'') ))  - - /# rncj|jD]$}t|tsJ|j&yrw)rrrrrs rlrz(FusedSchedulerNode.swap_pw_red_dimensions,{{Gg}5 55  ) ) +#rncttdd|jD}t|dk(ryt |}|S)Nc3|K|]4}|js|jr|j6ywrwr r"rrs rlrz4FusedSchedulerNode.estimate_flops..6 0'')T^^-='') 0:<rrfilterrrrrkfpsrs rlrz!FusedSchedulerNode.estimate_flopsK  $ 0    s8q=#h rnc|jryd}|jD]`}t|tsy|;t |t |j dk7rt jdy|j d}bd}|Jt||jcxk(r|jk(rnn|j|}|s%t jd|jytxjdz c_ t jd|j||jD]%}t|tsJ|j|'t|y) z@ Return true if a loop reordering is performed. FNrz1Can not reorder fused node due to different sizeszODont reordering fused node %s because we can not decide the suitable loop orderrz-Reorder loops for fused node %s with order %sT)r rrrrrYrrrrrrr,rrr)rkrrrrrs rlrz,FusedSchedulerNode.reorder_loops_by_dep_pairs?     [[Ee]3%% *;uU\\RS_?U*U!''GaJ! %%% z?h// E93E3E E ;;IFI  # #a  ##q(# ;T]]_i [[Ee]3 33  & &y 1! (-rnct||t|||g|_t |dj |_y)Nc4t|jSrw)r rrs rlrvz-FusedSchedulerNode.__init__..ss1>>3C/Drnr$)r>rxrrrr)rkrrrGs rlrxzFusedSchedulerNode.__init__s8 #i0%' %DEKK rnczdj|jDcgc]}|jc}Scc}wN_rrrrkrJs rlrzFusedSchedulerNode.get_name-xxt{{;{!{;<<;8c<|jdjSrrrrjs rlrz!FusedSchedulerNode.get_first_name{{1~&&((rnc|tj|jDcgc]}|jc}Scc}wrwrrrrrs rlrz#FusedSchedulerNode.get_buffer_names2 !L 1!"4"4"6 !LMM!L9cjg}|jD]!}|j|j#|Srwrrrzrkr-rs rlrzzFusedSchedulerNode.get_outputs/(*KKD MM$**, -  rnc ~t|jDcgc]+\}}|jd|d|j-}}}|jdj}||j |j tjdj|jdScc}}w)Nz.snodes[z] = rrr') rrrr/rrrr+r+rr)rkrrrs rlrz"FusedSchedulerNode.debug_str_extras%T[[1 14}}xs%0@/A B1  {{1~""   LL335 6tyy/668&AA s0B9ch|jDcgc]}|j}}|d|Scc}w)Nz , snodes: )rr)rkr snodes_strs rlrz"FusedSchedulerNode.debug_str_shorts:9=Ed**, Ez*..Fs/ct|||t}t|jD]/}|j|||j |j 1yrw)r>rrrrupdater`)rkrrrrGs rlrz!FusedSchedulerNode.set_last_usage sW 24FG0:|T[[)D    35G H  & &t 7*rnc|tj|jDcgc]}|jc}Scc}wrw)rrrrrs rlrz$FusedSchedulerNode.used_buffer_namess2!MA!"5"5"7!MNN!Mr c|tj|jDcgc]}|jc}Scc}wrw)rrrrrs rlrz/FusedSchedulerNode.used_or_aliased_buffer_namess78< D 1a,,. D  Dr c|jSrwrrjs rlrzFusedSchedulerNode.get_nodes rrncTt|jd|jdS)Nz(nodes=r~rrjs rlrzFusedSchedulerNode.__repr__#s't*%%&gdmmo->a@@rnc:td|jDS)Nc3<K|]}|jywrw)rrs rlrz2FusedSchedulerNode.is_reduction..(s9[1>>#[rrrrjs rlrzFusedSchedulerNode.is_reduction&s9T[[999rnc:td|jDS)Nc3<K|]}|jywrw)rrs rlrz6FusedSchedulerNode.is_native_matmul..,s=A1%%'rrrjs rlrz#FusedSchedulerNode.is_native_matmul*s====rnc:td|jDS)Nc3<K|]}|jywrw)rrs rlrz3FusedSchedulerNode.is_split_scan..0s:k1??$krrrjs rlrz FusedSchedulerNode.is_split_scan.s:dkk:::rnc:td|jDS)Nc3<K|]}|jywrwrrs rlrz1FusedSchedulerNode.is_template..4s8Kq1==?Krrrjs rlr zFusedSchedulerNode.is_template2s8DKK888rncj|jD]$}|js|jcSyrw)rr rrkrs rlrz$FusedSchedulerNode.get_template_node6s0KKD!--// rnc |jdSr)rrjs rlrzFusedSchedulerNode.get_device=szz!}rnc:td|jDS)Nc3<K|]}|jywrw)rrs rlrz>FusedSchedulerNode.has_aliasing_or_mutation..BsEA1--/rrrjs rlrz+FusedSchedulerNode.has_aliasing_or_mutation@sEEEErnctrwNotImplementedError)rkrs rlrz'FusedSchedulerNode.update_mutated_namesF!!rnctrwr*)rkrs rlrzFusedSchedulerNode.add_fake_depIr,rnctrwr*r&s rlr(zFusedSchedulerNode.can_inplaceLr,rnc|j}djd|jD}t}|j |dt |j d|d|dt|jjd|dt|jd|d t|jj|jz d|d |jd|d |jd|d |j5|jD]!}|j |j!# d d d |j#d |j |j%|j-j/S#1swYXxYw#t&$rt(j+ddYLwxYw)rr$c3FK|]}t|jywrw)rryrs rlrz/FusedSchedulerNode.debug_str..RsF+QQ 0 0+s!r#rrrrrrrrz.outputs = [ Nr%rTr)rrrrRrrryr)rrrqrrarbr+rzr/r&rrrrr,r)rkr node_typestrrrs rlr/zFusedSchedulerNode.debug_strOs}}xxF$++FF  bd  Q|n-j))00123WT%<%<=>?74#3#3#9#9D.nsG;4t,,.;r)rrr>r*rs rlr*z#FusedSchedulerNode.has_side_effectsks0 ;; "G4;;GG Gw'))rnrr_rr_rrrrTrr)rrrrrrUrRrrrirrrVr)r torch.devicer)rr3rrUr)%ryrzr{rr|r}rrrrrIrrrxrrrrzrrrrrrrrrrr rrrrrr(r/r*rSrTs@rlrrs $# +% +.? +  + +, ")!).7) )VL ==)NN B/ 8#2 8HV 8  8OO  A::>>;;99 FF """*8**rnrcDeZdZdfd Z ddZddZddZxZS)FusedMixOrderReductionsc`tj|stj|sJ||}}||_||_t||j t|jt|jztj|j|_ yrw) rrrrr>rxrrrrrF)rkrrrGs rlrxz FusedMixOrderReductions.__init__ss 33E:$77> >> %5E    OOT%//"34tEOO._get_ancestors..s:Eqq{{Errrrrs rl_get_ancestorszAFusedMixOrderReductions.sub_node_can_fuse.._get_ancestorss ,C399:E:; ;rncBt}|jd|DS)Nc3<K|]}|jywrw)rrs rlrzZFusedMixOrderReductions.sub_node_can_fuse.._get_operation_names..sF1q446rr?r@s rl_get_operation_nameszGFusedMixOrderReductions.sub_node_can_fuse.._get_operation_namess",C399FFG Grn) count_bytes)rtuple[BaseSchedulerNode, ...]rr_) rr8rrrrrtypingcastr score_fusion_memoryrF)rkrr other_nodesrArDs rlsub_node_can_fusez)FusedMixOrderReductions.sub_node_can_fuses e%<===e%<=== ~~&&ueu&U  / /  #66u= < H0 H  H u~.1Ek1RR{+.BE5>.RR""$ $ {{T^^77uRW7Xzz rnct|tsR|j|j||jfxs(|j|j||jfS|j|j|j|j|jfxr/|j|j|jt Srw)rr8rKrrrrkothers rl can_fuse_withz%FusedMixOrderReductions.can_fuse_withs%!89)) EDJJ=J'' EDJJ=I J )) EKK$**ekk)BK((U[[%'J KrncT|jj}|jj|}t |t rX|j |j|j}|j |j|j}t ||S|j|j||jfr2|j |j|}t ||jS|j |j|}t |j|Srw) rrrrcrr8rrrrK)rkrNrbackend fused_node1 fused_node2r4s rl fuse_withz!FusedMixOrderReductions.fuse_withs&&(..,,V4 e4 5!,,tzz5;;?K!,,tzz5;;?K*; D D%%djj%$**G$\\$**e< .z4::FF$\\$**e< .tzz:FFrnr")rr_rr_rJrF)rNr_)ryrzr{rxrKrOrTrSrTs@rlr8r8rs6 =2  2 !2 3 2 h KGrnr8cfeZdZ dfd Ze ddZddZd dZd dZxZ S) $FusedExternTritonKernelSchedulerNodec,t|jtjsJt j t t||g}t|%||||_ ||_ |jj|_ |j|_ yrw)rrr+r@rGrHrr_r>rx kernel_nodefused_epiloguercrj)rkrrXrYrrGs rlrxz-FusedExternTritonKernelSchedulerNode.__init__s{ +**B,F,FGGGT"34{N6ST F+&,))33%-- rnc|j}t|jdk(sJ|jt t |jj }|jjt|t|||SNr) rrrqr@rrrrremoveNodeUserrV)rqrrroriginal_mutated_buffers rl epilogue_fusez2FusedExternTritonKernelSchedulerNode.epilogue_fuses{ OO 5++,111"+"7"7 e../ 0 5 5#  %%,,Xe_=3IueLLrnct|jjtjsJt|j jtj sJ|j jjsJtj|j jjdj}ddl m }|j|jg|\}}ddlm}||jg|}ddlm}||||} | j'} |j jj)||jj| fS)Nrrr)SIMDKernelFeatures)FusedUserDefinedTritonKernel)rrYrr+rrXr@rArBrrCrDtorch._inductor.codegen.simdrsget_tiling_and_scores,torch._inductor.codegen.simd_kernel_featuresratorch._inductor.codegen.tritonrbr;codegen_with_epilogue_fusion) rkrNrFrstilingrrakernel_featuresrbfused_user_kernelnew_kernel_srcs rlr;z,FusedExternTritonKernelSchedulerNode.codegens $--22B4E4EFFF$**//1K1KLLL$$66888 $**//<Q Y( (rnct|}jr|jrtjttjt|}t j t |j k(}|s|d|xr2tfdtj |j DS|jrkjr |dytjt|}|j}||jj|S|dyjrk|jr |dytjtj|}|jj||S|dytd) Nzforeach do not have same lengthc3\K|]#\}}jj||%ywrw)rr)rlrrrs rlrz6ForeachKernelSchedulerNode.can_fuse..8 s0)ADAq""++Aq1As),zXcandidate producer is a reduction, foreach ops cannot be fused with reductions currentlyFz5candidate producer is not dep of any foreach consumerzXcandidate consumer is a reduction, foreach ops cannot be fused with reductions currentlyz5candidate consumer has no dep in any foreach producerzXAt least one node passed to ForeachKernelSchedulerNode.can_fuse should be a foreach node)rr$rGrHrorrrrrrsrrrzr)rqrrrvwhy foreach_matchconsumer_subnodeproducer_subnodes ` rlrz#ForeachKernelSchedulerNode.can_fuse/ s(+    X%8%8%:{{#=xHH{{#=xHH0C4HHM 56 S)A)&  "$$&n{{#=xHH'@@J +))228=MNN G H  "$$&n{{#=xHH'@@J +))223CXNN G H f  rnc |js|jsJ|jr3tjt|}|j}|j }n2tjt|}|j}|j }d}d}|jr|jr|tjt|}tjt|}t |j|jDcgc]\}}tj||} }}n/|jrtjt|}|j|} g} |}d}|jD]A} | | ur*tj| |} | }| j| 1| j| Cn|jrtjt|}|j|} g} |}d}|jD]A} | | ur*tj|| } | }| j| 1| j| Cn td||j| ||||Scc}}w)NzTAt least one node passed to ForeachKernelSchedulerNode.fuse should be a foreach node)use_custom_partition_algo prev_node_1 prev_node_2enable_autotune)r$rGrHrorrrrrrrrzrrsrr)rqrrrvrrrrr}r~ fused_nodesrrnew_noders rlrrzForeachKernelSchedulerNode.fuse^ sS""$(;(;(===    {{#=xHH(0(J(J %&66O{{#=xHH(0(J(J %&66O     X%8%8%:{{#=xHH{{#=xHH AADAq#''1-A  "{{#=xHH'@@J K"KK ++166tXFH"*K&&x0&&t, ( "{{#=xHH'@@J K"KK ++166xFH"*K&&x0&&t, (!f     &?##+   Ks I?cni_i_||qt |||D]Z}|jj D]}|j|j <|jD]} |j| <\n<|_|_ d_ g_ jtjj|j|jgt!fdt!j"|j$|j$Djj&z _t)|j*|j*g_t-|j.|j.g_t)|j0|j0_t-|j2|j2_|j5rt7|t8sJ||} } nt7|t8sJ||} } | j:_j:j=| j:| j_| jD]} | j| <jD cic]'} | j>jAD]\} }| | )c}} } _|_!|djE}|sJ|tGjHdfff_%t!tLjNjP_)|_*ycc}} } w)Nc3ZK|]"}|jjvr|$ywrwrrs rlrz6ForeachKernelSchedulerNode.__init__.. s4 xxt'<'<'>>  rr combo_kernel)+rqrur>rxrrrrrrrrrr*rrrrrqrrrcrrdrarbr$rrorrrlitemsrrrExprrrfxNoder_r)rkrrrrrrrrHr foreach_noderrrvrrGs` rlrxz#ForeachKernelSchedulerNode.__init__ s  +"5 G Y / ,,22D37D%%dii03!446D.2D%%d+7 'DN DKDI)+DJ  ''22 ,,k.E.EF )//#66 8V8V ""))*  #!+"7"79N9N!OPDN +"7"79N9N!OPDN&).. 0N0N'D #'*.. 0N0N'D #%%'!+/IJJJ+6 j !+/IJJJ+6 j )33DN NN ! !*"6"6 7 , 9 9D "668*4!!$'9#'++@"-%:O:O:U:U:W$!Q1:W+@D *C&%%' v > :<>? !%((--02 .@s ,L0c |Dcgc]}t|ts|}}|rStjdt ||Dcgc])}|j |j j +c}|Dcgc]}t|ts|}}|rtjdt ||Dcgc]}t|ts|}}|rtjdt ||Dcgc]$}t|ttttfs|&}}|Dcgc]}t|ts|}}|rtjdt ||Dcgc]}t|tr|}}|Dcgc]}|js|} }| r tjdt | | |Dcgc] }|| vs| }}tjra|Dcgc]}|js|} }| rtjdt | |Dcgc]}|jr|}}|Scc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}w)Nz/ComboKernels: %d external nodes are filtered %sz+ComboKernels: %d grouped nodes are filteredz;ComboKernels: %d FusedMixOrderReductions nodes are filteredz+ComboKernels: %d foreach nodes are filteredz0ComboKernels: %d template nodes are filtered: %szCComboKernels: %d reduction nodes are filtered (pointwise_only mode))rrrrrrrWrr8rAror r(combo_kernels_pointwise_onlyr) rqrrJexternrgrouped mix_orderfiltered_nodes foreach_nodestemplate_nodesreduction_nodess rlcombinable_nodesz+ForeachKernelSchedulerNode.combinable_nodes sv#OUj4M&N!UO  IIAF 5;UVTtyy?T&&(VU  $Kez!5I'J1eK  II=G  !&P1A7N)OQ P  IIMI   *-(+     & %!A7Q)RA~    IICSEW X% %!Z;U-VA~  &4G^q}}!^G  IIBN#  &4O^q7N!^O  . .*8M.QANNI>3J Jc |j}g}d}t|Dcgc]0}|D])}t|tr|j D]}|+2c}}}}|D]}t t } |D][}|j} | r| jdk(s| jdk(r4|j|zrH| | j|]| jD]} t t } | D]0}| |jj|dj|2| jD];} |jtdt!| |Dcgc] }| |||z c}=|Scc}}}wcc}w)zS Returns a list of lists of nodes that are to be grouped together. mpsrr)_topological_sort_nodesrrr8rrrrrrrrnode_to_streamrrrr)r sorted_nodes grouped_nodes max_num_nodesrrrexcluded_buffer_namesr device_groupsr device_nodes stream_groups stream_nodesrs rl&_default_group_nodes_for_combo_kernelszAForeachKernelSchedulerNode._default_group_nodes_for_combo_kernels/ s!88:   1;* )E!Dd$;< $ 5 5 7H !8 !) 2 "ED! *v{{e3v{{e7K))+.CCf%,,T2!. 4 4 6 DOPTDU (D!)":":">">tQ"GHOOPTU)$1$8$8$:L!((&+1c,.?%O%O)Q->?%O%; !7%":K @s 5E? F4Callable[[Scheduler], list[list[BaseSchedulerNode]]]!group_algorithm_for_combo_kernelsc|t_yrwror)custom_group_algorithms rl%set_group_algorithm_for_combo_kernelsz@ForeachKernelSchedulerNode.set_group_algorithm_for_combo_kernelsf s # #Drnc,tj|Srwrrs rlgroup_nodes_for_combo_kernelsz8ForeachKernelSchedulerNode.group_nodes_for_combo_kernelsn s*KKIVVrnctrwr*rjs rlrz#ForeachKernelSchedulerNode.mark_runt r,rnctrwr*rjs rlr;z"ForeachKernelSchedulerNode.codegenw r,rncyrJr~rjs rlr$z%ForeachKernelSchedulerNode.is_foreachz rrnc,t|jS)zeReturns a list of nodes which comprise the combo kernel. These nodes may be vertically fused.)rrrjs rlget_subkernel_nodesz.ForeachKernelSchedulerNode.get_subkernel_nodes} sDKK  rnctttjjd|jDS)zqReturns all nodes contained in this kernel, unpacking fused nodes into their constituent scheduler nodes.c3<K|]}|jywrw)rrs rlrz7ForeachKernelSchedulerNode.get_nodes.. s1UA!++-r)rrrrrrjs rlrz$ForeachKernelSchedulerNode.get_nodes s(IOO111U1UUVVrnc<|jdjSr)rrrjs rlrz)ForeachKernelSchedulerNode.get_first_name s{{1~,,..rnct|||jj|jD]}|j |yrw)rrr@rr)rkrrs rlrz/ForeachKernelSchedulerNode.prune_redundant_deps s8 d$68R8RSKKD  % %&8 9 rn)rrr_rr)rvr_rrrrr_rvr_rrv)rrr_rvr_rro)NNF)rrrrrrvrrrrrrvrrUrrrr)rrrlist[list[BaseSchedulerNode]])rrrrUrTrVrrrrRr)ryrzr{rrsrzr}rrrrxrrrrr|rrrr;r$rrrrrSrTs@rlroro s ) !) !&, , \> (> 4E> #> > J1504 %L/L/(L/$( L/ . L/ . L/L/ L/\?+? ??B00 &00h /&(/ T   WW &WW ""! W /:">: :rnroceZdZUdZded<eddZ d dfd ZddZddZ e ddZ dd Z e dd Z dd Ze dd Zdd ZddZeddZxZS)raC This is a "fake" scheduler node that represents a group of scheduler nodes that are meant to be *grouped* together (it does not allow another node to be scheduled in between its constituent nodes, nor does it allow another node to fuse into any of its constituent nodes). The way it does this is by maintaining its unmet dependencies as the union of its constituent nodes. Fusion will still happen among the nodes within each GroupedSchedulerNode. At codegen time, this scheduler node will be unpacked and codegen is called on each constituent node. rrc|djtfd|DsJ||}|D]}|j|j<!|j|j<|S)Nrc3:K|]}|juywrwr)rrrs rlrz.GroupedSchedulerNode.create.. sB644>>Y.6)rrrr)rqr grouped_snoderrs @rlcreatezGroupedSchedulerNode.create sr1I'' B6BBBBIv. E=JI ( ()9 :AN $$]%;%;%=>rncLt||t|||||_yrw)r>rxr temp_grouping)rkrrrrGs rlrxzGroupedSchedulerNode.__init__ s( #i0 +rnc6|jr |jS|jD])}||jj|j <+|jj|j =|jj |jS)z Do fusion among nodes within this GroupedSchedulerNode, and then unpack this GroupedSchedulerNode into regular nodes. )rrrrr fuse_nodes)rkrs rlunpackzGroupedSchedulerNode.unpack sq   ;; [[EBGDNN - -enn.> ?! NN - -dmmo >~~((55rnc|j|jj||jj |yrw)rrrrqr)rkfake_deps rlrz!GroupedSchedulerNode.add_fake_dep s5 T--77AB ##H-rnczdj|jDcgc]}|jc}Scc}wrrrs rlrzGroupedSchedulerNode.get_name rrc<|jdjSrrrjs rlrz#GroupedSchedulerNode.get_first_name r rnc|tj|jDcgc]}|jc}Scc}wrwr rs rlrz%GroupedSchedulerNode.get_buffer_names r r cjg}|jD]!}|j|j#|Srwrrs rlrzz GroupedSchedulerNode.get_outputs rrncttdd|jD}t|dk(ryt |}|S)Nc3|K|]4}|js|jr|j6ywrwrrs rlrz6GroupedSchedulerNode.estimate_flops.. rrrrrs rlrz#GroupedSchedulerNode.estimate_flops rrnc|jSrwrrjs rlrzGroupedSchedulerNode.get_nodes rrncX|jr|jdjSdSr)rrrjs rlrzGroupedSchedulerNode.get_device s$.2kkt{{1~((*CtCrncyrr~)rqrrrvs rlrzGroupedSchedulerNode.can_fuse rrn)rrrr)F)rrrrrrvrrUr)rr3rrUrRrr5rrrXr)ryrzr{rr|r}rrxrrrIrrrrzrrrrrSrTs@rlrr s $#$ + +( + +  + 6.==)NN "Drnrc  tjdfd }ttt t d}t |dkDr|Dcgc]}| c}t jr|j||Scc}w)z A heuristic to decide loop iteration orders. This has not been well tuned and may be something we should autotune. ct|dk(s|dk(rt|dk(|dk(SDcgc]}t||}}Dcgc]}t||}}tdt||D}tdt||D}||kDry||kDryt||Scc}wcc}w)Nrc3:K|]\}}|dk(xs||kywrNr~rsl_asl_bs rlrz5pick_loop_order..index_cmp.. ) 7VtDAI $ $7Vc3:K|]\}}|dk(xs||kywrr~rs rlrz5pick_loop_order..index_cmp.. rrr)rJabsrr) rbsl stride_len_a stride_len_ba_firstb_firstr~stride_lengthss rl index_cmpz"pick_loop_order..index_cmp s 8q=E!HMuQx1}eAh!m4 4.<<^rBqE ^ <-;<^rBqE ^ < 7:<7V   7:<7V   W  W 1ay#=>#33 .  E$5$55 .  -  rnc6|jjSrwr1rjs rlrzNodeUser.get_name r2rnc|j|jusJt|j|jxr |j|jxr |jSrw)rr]r(rrMs rlrLzNodeUser.merge sPyyEJJ&&& II    2!2!2 LL *U]]  rnNrS)rNobjectrrvrR)rNr]rr]) ryrzr{r|r(rr!r"rrLr~rnrlr]r] s3 ((KGTL $ rnr]c"tjSrw)r(r r~rnrl*used_non_deterministic_runtime_estimationsr' s  3 33rnct}|j}t|tjr|j t |jt |jzt |jzt|tjr$|j t|j|S| Jd||S)z=Get free symbols from a node's layout (size, stride, offset).z*Expect layout to be None but found layout=) rmaybe_get_layoutrr+Layoutrr"rstrideoffsetrCget_layout_symintsrX)rfree_symbol_usesr's rlr-r- s1;  " " $F&"))$  %6==) *6==) * fb;; <  # #$6v}}$E F ~T!KF8TT~ rnc"t|tr(tjd|jDS|j J|j j }|jd|j jD|S)z Gets symbols used in a scheduler node, including free symbols from the node's operations and layout symints from outputs. c32K|]}t|ywrwget_scheduler_node_symbol_uses)rrs rlrz1get_scheduler_node_symbol_uses.. s M,U3c32K|]}t|ywrw)r-)rir_nodes rlrz1get_scheduler_node_symbol_uses.. s M5L' W %5Lr3) rrrrrrget_free_symbol_usesrrz)rr.s rlr2r2 s$*+!z|!! M M   99  yy557 MTYY5J5J5L M rncv|j}||j |jStjSz4Check per-template flag, fall back to global config.)rallow_epilogue_fusionr(epilogue_fusionrtbs rl_is_epilogue_fusion_enabledr= 8  ( ( *B ~"22>'''  ! !!rncv|j}||j |jStjSr8)rallow_prologue_fusionr(prologue_fusionr;s rl_is_prologue_fusion_enabledrB r>rncb|jxr|j xr t|Srw)r r=rs rlis_epilogue_fusionrD 4  /!!# # / ' .rncb|jxr|j xr t|Srw)r rBrs rlis_prologue_fusionrG rErnc6t||xs t||Srw)rDrGrs rlis_template_fusionrI s eU + O/A%/OOrnc"t||r|S|Srw)rDrs rltemplate_fusion_pw_noderK s&ue45?%?rnceZdZdZdjdZdjfd ZdkdZdldZdmdZdndZ dodZ e dpd Z e jdqd Z dld Zdrd Zdsd ZdldZdldZdldZdldZdtdZ dudZdldZdvdZdwdZdldZdldZdldZdudZdldZ dxdZ dy dzdZ d{dZ! d|dZ"dld Z# d}d!Z$d~d"Z% dy dd#Z& dd$Z'dd%Z( dd&Z) dd'Z* dd(Z+ dd)Z, dd*Z- dd+Z. dd,Z/dydd-Z0dd.Z1 dd/Z2 dd0Z3 dd1Z4 dd2Z5 dd3Z6 dd4Z7 dd5Z8 dd6Z9dd7Z: dd8Z; dd9Z< d dd:Z= dd;Z> dd<Z?dd=Z@ dd>ZAddd?ZB d dd@ZC ddAZD ddBZE ddCZF ddDZGdldEZHdldFZIdldGZJ ddHZKddIZLddJZMddKZN ddLZOddMZPeQddNZR ddOZS ddPZT ddQZU ddRZV ddSZW ddTZX dudUZY dudVZZ dudWZ[ ddXZ\ ddYZ]ddZZ^dld[Z_ dd\Z` dd]Za dd^Zbdld_Zcdd`Zd ddaZeddbZfddcZgdlddZhe ddeZie ddfZjddgZkdldhZlddiZmxZnS)rz A Scheduler is a graph of BaseSchedulerNodes. It is responsible for optimizations such as fusion, reorder, and graph partition. cftd5|j|dddy#1swYyxYw)NzScheduler.__init__)r_initrkrs rlrxzScheduler.__init__ s# . / JJu 0 / /s'0c  t tj_i_t t_tj_ t_ tgtjjjtjj jtjj"j_|Dcgc]}j'|c}_d_d_j/j$j1tjj jj(D]}|j3d_j7_j(Dcic]}|j;|c}_j(Dcic](}|j?D]}|j;|*c}}_ j<jC_"i_#i_$t_%tMjNj(j@jD_jQjSj(_jUj(Dcic]}|j;|c}_"jWjYtZxj\t_j(z c_.ddl0m1}m2}|j(t_j(_3jijSj(_ttjtltlf_7tpjr$tqjrj(_tpjtr'ddl;m<}|j{jWi_>i_?d_@i_Ajjj(_tpj$tqjj(_tdj(DrjUjjtpjstpjr.q s  " tA B"rz#Scheduler.create_combo_kernel_nodesTlog_pt2_compile_eventlog_waitcounter) num_ck_nodes)reorder_for_peak_memory)1assign_memory_planning_info_for_scheduler_buffers)6align_runtime_estimations_across_all_distributed_ranks)trace_structuredartifactc dddS)N#scheduler_nodes_before_comm_overlapstring)rencodingr~r~rnrlrvz!Scheduler._init.. s E$,)rnc djtjDcgc]0\}}d|d|jzd|j z2c}}Scc}}w)Nz zsnode[r%z buffer_names:)rrrr/r)rrrks rlrvz!Scheduler._init.. sqv{{ )2$**(= )>1%QCqMkkm, .q/A/A/C.DEF)> (s5A" ) metadata_fn payload_fn graph_statsc^jjtjdS)N)graph_idnum_nodes_before_fusionnum_nodes_after_fusion)post_grad_graph_idnum_orig_nodesrrrjsrlrvz!Scheduler._init.. s% 33+/+>+>*-djj/rn)zr>rxrZrrbackendsr_post_grad_graph_counterrjrcount_graph_partition_counterrr>rr constantstorchbind_constantsrcreate_scheduler_noder previous_node current_nodeupdate_zero_dim_cpu_tensorrrdefault_device_contextget_donated_buffersr?rrurzr@copyrrrgseen_template_fusionsr'decide_global_ordering_of_commsr]topological_sort_scheduledead_node_eliminationcompute_ancestorscompute_input_distancesr,ir_nodes_pre_fusionrtorch._inductor.debugrQrRrkcreate_foreach_nodesrr logged_slow_fusionr(_pre_fusion_custom_passdistributed_max_autotune_gemmrrSschedulerbuff_to_stream_multi_stream_nodesstream_idx_to_user_obj_idx_populate_stream_assignmentsr_post_fusion_custom_passrrfinalize_multi_template_buffersmax_autotune_gemm max_autotunerrrselect_algorithmPrecompileThreadPoolshutdown_instance combo_kernelsrcreate_combo_kernel_nodes_enforce_conditional_orderingrYmemoryget_output_names deterministic reorder_for_compute_comm_overlaprZr'r)6runtime_estimations_align_across_all_distributed_ranksr rrSrr[reorder_sink_verbose_loggingtorch._loggingr\$reorder_compute_and_comm_for_overlapprocess_grouped_nodesgraph_partitionrr^%reorder_for_reducing_graph_partitions&maybe_reorder_for_minimizing_partition,reorder_for_partition_with_simple_dependencycompute_last_usage test_configstrack_memory_lifecycleinsert_memory_check_nodesr graph_diagramdebug_draw_graphbuffer_names_to_freeorigin_to_index_current_stream_ctxradd_row removed_ops)rkrrrrrQrRrSrYrZhas_collectivesr[r\rGs` rlrNzScheduler._init s  <> "&'?"@(1(9%5?\!&0 %%**, ""'') ,,113 ' #>CCUd003UC 7;6: '') ##**177+<+<+A+A+CDJJD OO <@#  $ $ & #&*ZZ; %/AJJL!OZ;  -1JJ8 ,6D$BRBRBT3CLLNC BTNJ8 AE@Q@Q@V@V@X3513 L " :: JJ     # #  !!#33DJJ?  ""$<@JJ"GJq1::<?J"G   $$& ##s4::6#O$**%!$**o !!#33DJJ? ",U38_"="?  ) ) 577 CDJ  / / . ) )$ /  " " $=?.0). :<' ))+__TZZ0  * * 688DDJ       & & (  ,,.  $ $(;(;&( OO , , A A S S U   5&* $ ..D.A  **,  ) ) 70   ''177//44671773356 DJ##(O(O11UAJJ 0 0 ;< WW<<#PP#( JJD$TYY/*.'#K4::V88; !  CCDJJODJ ""$  " " (( CCDDTZZPDJJJ4::VDJ ! ?? ! ! . . E E  * * ,4::& djj) 6@\!:<GK '//  -7LAD; 8 H#Hhs$5c$?c).-c.(c4 c99dc i}tjjD]d}ttjj|tj s9t |tjj|d||<f|S)N)r)rZrgraph_inputs_originalrr+ DonatedBufferr])rkname_to_donated_bufrs rlrwzScheduler.get_donated_buffers sk GG11D!''77=r?O?OP,BGG11$7 $-#D)2#"rncv ddlm i}tjd}|jD]} }|j D|j j }|(||vrt|}|||<||j|<||}||j|<|jD]}||j|<t fd|jjDrtd|jDd}||jD]z}|j } |j t| t j"s;t| j$t j&s`t!j&|| _|t fd|jjD|_y)a=Populate node_to_stream and buff_to_stream from IR node stream_idx. Reads the stream_idx field set on IR nodes during lowering to determine which stream each scheduler node should run on. This field is propagated from 'custom.stream' FX node metadata via IRNode.current_stream_idx(). r)DEFAULT_STREAM_IDXNc3(K|] }|k7 ywrwr~)rrtrs rlrz9Scheduler._populate_stream_assignments..' sM0L1q&&0Lc3^K|]%}|j|j'ywrwrrs rlrz9Scheduler._populate_stream_assignments..) s"RAq||~7Qs--rc3(K|] }|k7 ywrwr~)r stream_idxrs rlrz9Scheduler._populate_stream_assignments..7 s' :  , ,:r)stream_constantsrrrnrrget_stream_idxrrrrrrrrrr+Bufferr'r?r) rkuser_obj_to_stream_idxstream_idx_counterrr user_obj_idxnew_stream_idxrrr5rs @rlrz&Scheduler._populate_stream_assignments s 924&__Q/JJD+Jyy$#yy779 +#+AA)-.@)A?M.|>:D AS H +  6 Irnctjtjr8tj d||j D]}|j yy)Nz%s:)r isEnabledForloggingINFOrrr)rklabelrs rldebug_print_nodeszScheduler.debug_print_nodes^ sA   GLL ) HHUE "   "# *rnc6|jJd|jr t||St|tj tj fr t||St|tjr t||St|)Nz2All nodes passed to scheduling must have an origin) rWis_no_oprArr+rrarrrr+r%s rlrrzScheduler.create_scheduler_noded s!- @ - ==?)$5 5 r00"2C2CD E t, , boo .,T48 8%d+ +rnct}g}|jj}tjj j D]}|Dcgc]%}||vrt|j|ts|'}}|s6|j||Dcgc]}|j|}}tjdkD}t||d|}|j||D]}||j|<|jD cgc]} | j!|vs| c} t#|z|_ycc}wcc}wcc} w)NrFrr)rrrrZrlistsrrrurArr(combo_kernels_autotunerorrrr) rkremoved_node_namesfe_nodeskept_node_namesnamesrrrfe_noders rlrzScheduler.create_foreach_nodesq sL.8l11668WW]]))+E"!D?*"4#4#4T#:eZdZdZ d ddZddZd fd Zy) 1Scheduler.compute_dependencies..DedupListan This data structure behaves like a list except it makes sure the elements remain unique. Normally one could use a OrderedSet/dict for this purpose however the list in question gets elements appended as it is being iterated over which means that we need to keep the list semantics. Nc@|xsg|_|xs t|_yrw)rr membership)rkrrs rlrxz:Scheduler.compute_dependencies..DedupList.__init__ s #[b ","< rnc||jvry|jj||jj|yrw)rrrr)rk node_users rlrz8Scheduler.compute_dependencies..DedupList.append s5/ !!),##I.rnctj|j|j}|j|jDcgc]}||jvs|c}z}||Scc}wrw)rrrr)rkrNnew_membershiprJ new_items DedupLists rl__add__z9Scheduler.compute_dependencies..DedupList.__add__ sg!+!1!1$//5CSCS!T JJ${{**!at.FA{* !N;;*s A+A+r)rzlist[_T] | NonerzOrderedSet[_T] | NonerrU)rrarrU)rN DedupList[_T]rr)ryrzr{rrxrr)rsrlrr s; *.48 =& =2 =  = /  .rename s,D)))d33A677HrnFcP|jt|||yrw)rr]) used_by_namer5r(r name_to_usersrs rladd_userz0Scheduler.compute_dependencies..add_user s) &. / 6 6K9 rnNc|jSrwrrs rlrvz0Scheduler.compute_dependencies..sAFFrnr$Tz scheduling %s) unbacked_onlyc|jSrwrrs rlrvz0Scheduler.compute_dependencies..s!&&rnz not in )r mutating_bufr)r)rzscheduling output %sz+scheduling output %s for unbacked symint %srS'z': r$rTzBUFFER USER LIST z===== AFTER SCHEDULING ===== %s)rr rr )FF) rr r5rr(rvrrvrrU)Er rarrrrzrrrr'r+r?rr(rZrrrrrr" TensorBoxrr)get_unbacked_symbol_defsSymbolrrr6rurr5rrrrr4rr*rr_r6additional_buffer_depsadditional_star_depsrrr(rrgrrrrG graph_outputsrrmutated_inputsrrprmutated_input_idxsrMr?rRrr+r,rcompute_dependencies_log)*rkrbuf1 buf1_name buf2_namelist1list2combinedr%runbacked_symbol_to_origin_nodevalfsrtsym_sizehas_non_input_unbacked_defsunbacked_symbol_defsunbacked_symbol_usesr~rr node_modealt_namer.out_buf other_nameis_aliasadd_deprHrrrr inp_nameslogbufrrrr rrrs*` @@@rlr]zScheduler.compute_dependencies s}  <  <@@K?V?V @ JJD((* MMO tyy//?D,,./!3!%!1!1!3I M1i=6P -i 8 -i 8#(5=#0C -c 2e ;#0#5#>5= c 2 $1 #m33@3K i03@3K i0"4+< !&!  6      JL& 77''..0C#uzz***B9=226+C.(+||~S~!Auzz9RA~S!Ann=A6r:-"1',#JJD99( (($* 224:J$ *!!U\\222/3+::8< 215* JJD IIotyy 1*yy,,,'-II222F(($ .A >>#X&D%EF>>K#'#4#4Q#7#C#C#EC --gclln.EF$F .D$$++,1 d&6&6&=&=!>??S?sI.HH  '')3,,./1444 # 1 1 3H%h/HXt,%%ghY&GH -h 7 = ===?dmmo=$)$))5FGGG'+yy'<'<'>G)0)9)9);J)/ );J(073F3F3H'HH -- '$.1408L!"%ZtD%(? !> !4*B7799$--/J$5!!''4==?D"QR K 7777 H$6!!''"23I ((..!$0TYYd.>.>t.DE/  % %d&;&; <'') # 1 1 3H>AllnD))&*:;69llnD))(3//33HhG++CLLN;!4*ar002H II,h 7 Xz'(*;< =3 'ww,,111EA >>#X&D&I&I&K%LM>;1==q=(,(9(9!(<(M(M(OHII M ( ! %Xz'(:K/LM )P F-))Dqww+++z'$-89&&**40***z'$-89 *,5QWW5I5I5N5N5P+Q +QKE4D%K+Q  )*(>(>& (>IdO(>& " JJD'') mCLLN;AAB*//D  ' ' - 7 7 d8K8Q8Q R0 ! c'--/JC/4{{;{!{; #c%23!0  c  "))+ &&';< &&'I3OKTX & "<!s65i4i42 i9 i?j j5j j  j c ddlm}m}m}m}t t jjj}|j|}tjjjs|jjt t jj!}|j||\}} } t#t%jD cgc]} ggfc} |D]}} | j&dk(r| j(dk(r"| j*j-} | j.dj1| | j2dj1| ddlm} |  dfd } g}t9jD]H\}}|j1||j1| ||t%jdz k(J|_ ycc} w)Nr)rZcompute_memory_timelineFreeableInputBufferget_freeable_input_bufr)register_check_mem_opcX|d}|d}|||g}tjttjdtj j jjg|d}dj|j|_ t|S)Nrrrrc$||d|d|ddfS)Nrrr)alivedead is_final_stepr~) tensor_argsrs rlrvzWScheduler.insert_memory_check_nodes..construct_mem_check_node..s(!.q!1 -a 0)6q)9Crn)r'r=r#nontensor_argsunflatten_args mem_check_) r+MemoryCheckKernelr?rrr_inductor_debugcheck_memory_stepdefaultrrrr)step_idxr"expected_newly_aliveexpected_newly_deadr$rrkstep_allocs_deallocss rlconstruct_mem_check_nodezEScheduler.insert_memory_check_nodes..construct_mem_check_nodes$8#A!#D "6x"@"C 24GWN''!e)<=yy00BBJJ-  D%/tzz(/C/L/L/N.O"PD ,T48 8rn)r")r+r r"rvrr)rrZrrrrrZrrrrrrr(rYr@rrr size_alloc size_freer]r start_steprend_step#torch._inductor.runtime.debug_utilsrr)rkrZrrrrname_to_freeable_input_bufr buf_info_listrbuf_inforrr/ new_nodesrrr.s` @rlrz#Scheduler.insert_memory_check_nodess  )31773G3G3L3L3N(O "4::| < #%%== = D,, *4AGG4L4L4N)O 5 JJ &   q!$C O4C 4RH4C &H""a'H,>,>!,C//1H !4!4 5a 8 ? ? I !2!2 3A 6 = =h G& N 9 9*. 9 & 92  ,GAt   T "   (1DJJRS@S;SU -  eC s3 Hc| tjsyg}t|jD]}dd d}|j D]}t fd|j D}|r\tjd|jtjjj|jd}|j xr| }|s|j|tjd|jtjj j|j|j"j$D]}|j&|j(vs|j(|j&j }|Dcgc]0}|j*j|jk7s/|2c}|j(|j&_t-t||_|jD]}|j/ycc}w) z0 Remove any nodes without users Ncr|jxs*|jtjjvSrw)rrrZrr)r.s rlcan_eliminate_userz;Scheduler.dead_node_elimination..can_eliminate_users&||Tt}}!'':T:T'TTrnFc3.K|] }|ywrwr~)rur;s rlrz2Scheduler.dead_node_elimination..s#M9a$6q$99zremoved dead buffer: %sTzremoved dead operation: %s)r.r]rrv)r(use_dcerrrzrrrrrrZrr?rr*rrrrrr@rrr) rk updated_nodesractive_buffersr can_eliminaterHrr=r;s @rlr|zScheduler.dead_node_eliminations~~   TZZ(D U#N'') ##M399#M M  II7HGG++// ?%)N *!% 5 5 77N.visit../saffrnr$)rr)rqrr)rrrur-seenvisits rlrJz2Scheduler.topological_sort_schedule..visit,sa} !!"6"6.Zs&XZc))#.??AZs'*c3<K|]}j|ywrwr)rrrks rlrz1Scheduler._get_unmet_dep_nodes..[sQ=at66q9=) rrrrrArrrqrr RuntimeErrorrr)rkr unmet_depsr unmet_dep_opss` rl_get_unmet_dep_nodeszScheduler._get_unmet_dep_nodesHs&0l  )&"$   //sxx(0=d5k]!L YZX JQ=QQRRrnczg}tj|jd}i}|jD]P}|j|}t |||<|D]*}|j |g}|j ||||<,R|jD cgc] \}} | dk(s |} }} | rx|j | | D]7}|j |gD]} || xxdzcc<|j|9|jD cgc] \}} | dk(s |} }} | rx|rJd|Scc} }wcc} }w)zU Sort nodes by their topological order, return a list of node lists. rrzTopological sort failed!) rfromkeysrrXrrrrr) rkrrchildrenrrrcrrzero_deg_nodesr.s rlrz!Scheduler._topological_sort_nodes]s4 djj!,#%JJD,,T2Dd)E$KLLb) ! ). @ 1a! @ LL (#$LLB/D$K1$K0 ! $-2KKMDMDAqQ!VaMND 444y AEs D1%D1 D7D7cji}|jD]w}t}|jD]B}|j|jj }|j ||||z}D|||j<||_yt|jD]\}}||_ ||_ y)z. Populate each node.ancestors N) rrrqr@rrrrrrrcrd)rkname_to_ancestorsrrr dep_node_namers rlr}zScheduler.compute_ancestorsws 9;JJD)3I.. $ 0 0 : K K M  m,.}== /2; dmmo .&DN%TZZ0KE4"DN"DN1rnci}i}|jD]}|jsd}d}n|jDcgc]/}||j|jj dz1}}|jDcgc]/}||j|jj dz1}}t |}t |}|||j<|||j<||_||_ ycc}wcc}w)z Populate each node's min/max_input_distance with the depth from graph inputs, measured as dependency hops before fusion. Nodes whose dependencies are all satisfied by graph inputs/constants have depth 0. rrN) rrqr@rrrrrrarb) rkname_to_min_distancename_to_max_distancermin_distmax_distr dep_min_dists dep_max_distss rlr~z!Scheduler.compute_input_distancess! 02/1JJD** $66! 7))9)9#(()C)T)T)VW6! $66! 7))9)9#(()C)T)T)VW6! }-}-4<  14<  1&.D #&.D #) ! !s 4C:74C?cHtjsy|jD]}t|tt fr#|j stjdk7r=|jD]3}t|tr|jr$|j5y)Nhalide) r(rerrrrrU cpu_backendrr r)rkrrs rlrzScheduler.merge_loopss|00 JJDd]4F$GHKKMf&8&8H&D)!%75;L;L;N!!# *rnctddd5tdD]}t|}tj d|dz||j |d}t|}tj d |dz||||k(s|dk(sltj d |dznt jst jr|j |d}|cd d d S#1swYy xYw) zB Combine eligible nodes into FusedSchedulerNodes. zScheduler.fused_nodesTrUr z/===== attempting fusion (%d/10): %d nodes =====rF)is_reorder_roundz=completed fusion round (%d/10): fused %d nodes into %d nodes z+===== fusion complete (%d iterations) =====N) rrrrrfuse_nodes_oncer(reloop_index_inversion_in_fusion)rkrrold_lennew_lens rlrzScheduler.fuse_nodess #4QU 2Ye*  EE ,,UU,Ke*  TE  g%A$$Eq1u',1188,,UT,J;   sA7C!AC!!C*cg}|jD]4}|jt|tr|j n|g6||_y)zA Unpack GroupedSchedulerNode into regular nodes. N)rrrrr)rkr8rs rlrzScheduler.process_grouped_nodessE.0 JJD   !+D2F!G dV  rnct|dkDsJ|dj}||_|j|}t ddd5|j |cdddS#1swYyxYw) Benchmark fused list of nodes and return the execution time in milliseconds on randomly generated inputs. rbenchmark_fused_nodesTcompile_time_autotune_time_us)rVdynamo_compile_column_usN)rrrrcrrt)rkrrrQs rlrtzScheduler.benchmark_fused_nodesso5zA~~q$$&$""6*  #"&%D 007    s A%%A.ct|dkDsJ|dj}||_|j|}t d5|j |||cdddS#1swYyxYw)rsrgenerate_kernel_code_from_nodes hint_overrideN)rrrrcrrx)rkrbenchmark_kernelrzrrQs rlrxz)Scheduler.generate_kernel_code_from_nodessn5zA~~q$$&$""6* ; <::'};= < rtA| |j>|jB|_!|jE||||)ycc} } w#1swYxYw)a Finalize a backing choice for MultiTemplateBuffers which did not already have a choice finalized through fusion. In the case of an extern choice, this will result in replacing the SchedulerNode. If a MultiTemplateBuffer did not have any fusion opportunities, finalizing a choice will force completion of compilation and benchmarking. c3|K|]4}t|tjjjr|6ywrw)rrrrExternKernelCaller)rtimings rlrz.`s6*E) & % @ @ S S #*ErzZNo extern kernel detected to fallback to when layout constraints fail for Triton templatesNryc |dSr[r~rs rlrvz;Scheduler.finalize_multi_template_buffers..s qQRtrnr$r)#rrrrrr+MultiTemplateBufferr(r%force_extern_kernel_in_multi_templateget_min_choicerchoice_timingsrrrrrrmulti_kernel_hintsrrfinalize_as_triton_callersfinalize_as_triton_callerrcurrent_originsr_ output_noder StorageBoxOperationBuffer origin_noder:r' _replace_node)rkrrrmin_node_unfusedrchoicecallershinttimingsrrtriton_timings out_tensorbox out_storage out_buffers rlrz)Scheduler.finalize_multi_template_buffersMs!,GAt$ .: 2114"YY **PP*4*C*C*E'$a'+*4*C*C*E ($$OO&&?? ==jI&0&?&?&AF) & % @ @ S S 4: 0 % 'B *"EOO$D$D$W$W y  $OO&&?? 00NP(8 %+$=$=D&0&?&?d&?&SG-4MMO.,;DAq#-a1I#J!"1,;+. &))=)=)?^%TUV%WF,2GDM%> <.rename_deps..sKdscjj)9:dr1r)rrgs rl rename_depsz,Scheduler._replace_node..rename_depssKdKK Krn)rrprrp)rrrrrurrrrrrrqrrrrrzr@rrcrdrr`) rkrrrrnew_scheduler_noder real_namernew_outold_outrgs @rlrzScheduler._replace_nodes "*j9!77 C* 1 -?$--/*3E 0??4#3#3#9#94;R;RSC 3377$GGyG.1hh +T L1<  1 11 -0;  * * 0 00 &&,!$  * * ,d.>.>.@! GW4;D  W--/ 0#MMGM ! (,~~$'+~~$'+~~$(,%rnc&td|DS)Nc3K|]q}t|jdxrU|jduxrEt|jjdxr#|jjjdk(syw)rN scatter_moder)r<rrrrs rlrz,Scheduler._any_atomic_add..sq  AFFF # 9d" 9 ^4 9 ((L8 9sA7A9)r)rk node_lists rl_any_atomic_addzScheduler._any_atomic_adds     rnc"|j|d|}tj|}tjj j }|jsd}||fS|jd|}t|tsJ||fS)NT)r{rztriton_) kernel_name source_code) rxrloadrr async_compile AsyncCompileuse_process_poolrrr)rkrrzsrc_codemodrfuts rlcompile_kernelzScheduler.compile_kernels77 D 8 x(55BBD --/C Sz &&9(&SCc<0 00Szrncd  !"#$%&'()*+tdfD}tjs|stj dSj r(t jtjr jsjrtj dSj}|djsJjdk(r(tjdk7rtj dSj}tt!j"||}j%|rtj dSddlmt++|djJdfd %|rtd fDr jdurjnj*t *tj,sJj/*rtj d Si$g"tj0D]A}*j3|t5j7d D]~\}} t |t8j:j<j>s5*jA|5"jC|gjE||jF dddtId} d} i} "D]V\}} } | | jK*jA|5jY|\}}|| |<|| kr|} |} dddX| *jZ|<t | t\sJ| $|<Dtj^tad*jbD}texr xr|tjfk#tIdtIdc'(d&#sR*j3*ji\&'t5j7tkjld }n*jbDcgc]}|df}}r(rjo|njo|\(}n4stj d Sjq(ts()ddl:m;}g"d}|D]\}}t |t\ss&ty|dr|jz*jzk7r?r |'(zk\rn\|dz }|tjfkDrnB*jA|5 "jC|gjE| dddt}"dk(rtj d Sd"#$%&'()*fd }tj|"ddSjE|jE|!jE| d !%+fd }tj| dS#1swYxYw#tL$rI}tNjQtRjTrtNjWdsdnd|Yd}~qd}~wwxYw#1swYxYwcc}w#|$r YdddwxYw#1swYxYw) If config.benchmark_fusion is False, always return True. Otherwise, return True if fusion can brings speedup. c3K|]>}|jxr(t|jtj@ywrw)r rrr+rrs rlrz.Scheduler.speedup_by_fusion..sE $ MMO J1..0"2H2HI J#sAATrrrCompilationErrorNc ttjtjr|||zkrFtj dj j t ||z|z dytj dj j t|||zz dyy)Nz9can fuse (benchmark): fusing %s with %s cause %sx speedup.3fz=cannot fuse (benchmark): fusing %s with %s cause %sx slowdown)rrrDEBUGrrrErF)ms_fusedrrrrs rl log_fusionz/Scheduler.speedup_by_fusion..log_fusions&&w}}5cCi'$$S..0..0"sSyH&.)s"% 7E!A   ! -~Fc |dSr[r~rs rlrvz-Scheduler.speedup_by_fusion..=s aPQdrnr$ryinfException in compiling %s: %srrc3<K|]}t|tywrw)rr)rr\s rlrz.Scheduler.speedup_by_fusion..hs%ASA 167ASrr) CantSplitallowed_prologue_inpsc td}d}i}rhrttjsJj j \Dcgc] }|dvr| c}t fdD]0\}}} ||j}n!s|j}|jnd}r>j!|5j#|\} } | ||<| |kr| }|}ddd}|k(xsz|zkD} |s| s|j|j$r |j&sJ|j$d} | j&} | j(}t+|j&| ||j,j.t1j2}|s/|}nr |r|zkrL|Jt4j6r|d<j9nj;|r|j<d<yy cc}w#t$rI}tjtjrtjdsdnd|Yd}~d}~wwxYw#1swYxYw) Nrrc|dSrr~)rJrs rlrvzKScheduler.speedup_by_fusion..benchmark_when_ready..snQqT&:rnr$rrrTF)rrr+rrrr)r-r precompilerrrrrrswap_as_triton_callerr} launchersn_regsn_spillsrbmreqr rCrr(rrr_choice_timings) min_ms_fusedms_fused_choice new_timings fut_choicerrh mod_fusedresrrpathfusible_choicecompiled_kernelr r should_fuse_epiloguebench_epiloguerrr:future_choicesget_choice_timings_async hint_override_best_fusion_choicer min_choicerr ms2_fusedrrks rlbenchmark_when_readyz9Scheduler.speedup_by_fusion..benchmark_when_readys$U| "& +%*ZAWAW*XXX%/%>%>%@N&0&?&?&AOJ +9&*8J%a=N:#*8&N &,&:&N 2@-FFI!!-"(--/C!/"+"3"3CNN,"&C&'==fE-1-L-L ) &.NHd 3;K/',6/7 28FE'&0N"Sy>&+AI+MM' >"--/#&==V]]B#B.1mmA.>O+:+A+AL-<-E-EN3A # # & , . & 6 6 0 7 7 ?40 428 %y2@|"|S#6',#)*D%100AP8>"==< #<<_M%;F 2248 }&.%!%227==A&,, ?2A z ! !!FEs*H4H $I. I+">I&&I+.I8 cddlm} d d dfD]}||jj d \t j r dyj d \t j r dyj d \t j r dyt drWzk\rOfjvr?jjftdjfd zkS#|$rYy $r}d t|vrYd}~y d}~wwxYw) Nr)NoTritonConfigsErrorrz%register spilling of the first kernelFz®ister spilling of the second kernelz%register spilling of the fused kernel slow_fusionc $zz dS)N) kernel1_pathkernel1_latency kernel2_pathkernel2_latencyfused_kernel_pathfused_kernel_latencyslow_down_ratior~)rrrpath1path2 path_fusedsrlrvzKScheduler.speedup_by_fusion..benchmark_when_ready..`s&053605365?8@3;sSy3I%rnLoop-carried variableT) )torch._inductor.runtime.triton_heuristicsrr-r}rBisinfr rrrrr )rrrrrrrrrrrfuture_and_mod_l1future_and_mod_l1_fusedfuture_and_mod_l2rrkrs @@@@@@rlrz9Scheduler.speedup_by_fusion..benchmark_when_ready-sA*!,)!,/2  ?JJL  "&!@!@)!,"JC zz#CD$!%!@!@)!,"JC zz#DE$+/+J+J/2,(Hj zz(+CD$xc20 >$c 1"EN$2I2II//33UENC(7?? $cCi//+! '.#a&8#s<E AE +5E !5E A3E E.E. E)(E))E.rt)rrrrrrrrUrV)@rr(benchmark_fusionrdrrr rrr+TritonTemplateBufferr$rrrrjrrrrtriton.compiler.errorsrrrrrrr)rrrrTritonTemplateCallerrrrrzrr-rrrrrrr}rrbenchmark_epilogue_fusionrchoicesr max_epilogue_benchmarked_choicesroperator itemgetterrtrrrcrr<rrru),rkrris_multi_template node_list_1 node_list_2node_list_fusedrzrrrrrrhrrrrnum_triton_callerschoice_timings_iterr\rrtriton_choices unfused_timerrrrrr:rrrrrrrrrrrrrs,``` @@@@@@@@@@@@@@@@@@rlspeedup_by_fusionzScheduler.speedup_by_fusions  U^   &&/@$$T* *    u668":Q:QR!! $$T* *oo' Q**, v ;;% F$6$6($B$$T* *oo' y{KHI    0$$T* *;u% #..0!!! " % 8=u~% " $557tCO#''),,.  j"*@*@A AA55jA#((// -QSN!'!:!: !+!:!:=!I!'(<(<(>N!SIFA% @ @ U U!#99&A&-- &!%!4!4$36CWCW"5""BA "T %U| CG 1?-FFI !!-"MMO$99&A)-)H)H%v*$/7 F+#l2+3L.4OBA2@(=H **=9!/3KLLLBQ0?U";X$==N!$%AKASAS%"  )*R&&R&&*Q*QQ % U|U5\HC15J+!+!:!:!<",";";"= C&,"((*0C0CA0F'# 8B7I7I&J7I!1v7I#&J'..{;33K@U'',,U33224=/)"a'#((//j !j !j !X --$nQ&7&:  !% 3 3K @  $ 3 3K @ &*&9&9/&J #F F P --09PQR9S. g BA"%!%227==A&,, ?2A z ! !!BAF'Kb%!!>= ! >=s`:1XX*1$Y? Z Z%$ZX' * Y<3>Y77Y<?Z Z"Z%!Z""Z%%Z/ c<|j|jS)z0Look up the node in Scheduler name_to_fused_node)rrr%s rlr2zScheduler.get_fused_nodeys&&t':':'<==rncLtjd|j|j|j}|j|k(sJ|j |j ||}|j ||j ||j||jj|jDcic]}|j|c}|jj|}|||j|<|Scc}w)Nzfusing %s with %s) rrrrrcrrr\rrrrrr)rkrrrrnode3rstream1s rlfuse_two_nodeszScheduler.fuse_two_nodes}s ,enn.>@PQ!!#!V+++  (--eU;5!5! &&U__EV'WEV e(;EV'WX%%))%0  )0D   & (XsD!c|j||r-|j||s|r|j|||yyNTF)rwill_fusion_create_cycler)rkrr speedup_fnrs rlfuse_if_speedupzScheduler.fuse_if_speedups? MM% '11%?   uk :rnc|rg}i}t}|D]}||vrt||dk\sJ||jd}t||dk(r|j||j \}} | |k(rt || sJ|} n||k(sJt || sJ| } |j| | ur|jr3|jj} | J|j| ||f|| <|j|| |j|s|j|t|D]l} || \}} |j|j|j|j|j|j|s\|j| n|D]} |j| |ryy)z Evaluate pending template fusions for a set of fusion candidate nodes. The fusion candidate nodes are pointwise nodes as potential epilogue or prologue fusions rrN)rrrrrrDrGr2rhrrrfrrr) rktemplate_fusion_candidatesrtemplate_futuresfuture_to_pending_fusionfusions_to_remove candidatepending_fusionrrrfcands rl"_evaluate_pending_template_fusionsz,Scheduler._evaluate_pending_template_fusionss)-/  %@J| 7 !;;6yABaGH">@ uI%-eU;;;$)M I----eU;;;$)M &&}5]J!((&--44A=(=$++A.3A92M,Q/++un&@&@+*--i8K8P""23'?'B$''''(<(<=''(<(<="..  &))$/4'*..q1'q)rnc dfd }|D]H\}}|||j|}j|}t||r||fjvrOj|||scj ||rvj ||} | j t| j ||| j} t||rY||fjvsJjj||ft||} | |vrg|| <|| j| n | |<| |<(| js6j||Ky)Ncbj|vsj|vrjj|jj|}|J|j\}}|j}j |dj |dj||usJj||usJ|rj ||rj ||j|vrj|vryyrw)r2rrrfrrr) rrr! node_key1 node_key2 is_speeduprpending_fusionsrks rlresolve_pending_fusionsz.resolve_pending_fusionss1 ##E*o=&&u-@!0!4!4''.#''(;(;E(BC"&111'5'F'F'H$ 9+77 ##It4##It4**95BBB**95BBB!|t'D'DUE'R##Iy+F+##E*o=&&u-@rn)rfrrrhr")r2rIryrrrrfrrhrrKrrer) rkpossible_fusion_pairsr*template_fusion_nodesrrlr+rr fusion_resr!template_pw_nodes ` ` ` rl_try_fusion_pairszScheduler._try_fusion_pairss G$ G$ G G82LE5 $E5 1''.E''.E#5%0ENd&@&@@}}u.33E5A!33E5A ))5%2$.$:$:##)00 &N*%7 %u~T5O5OOOO2266u~F+B5%+P(+3HHFH12BC-.>?FF~V1?.1?.!--##E5+>W2rnc@t}|jD]}|j\}}|j}||vs t ||r3|j ||j ||usJ|j ||usJ|j||||yrw)rrrrfrIrr2r)rkrr*seen_pair_speedup_fnr!r'r( is_speedup_fns rl_finish_pending_fusionsz!Scheduler._finish_pending_fusions=s @J|.446N#1#B#B#D Iy*66M 448J99 $ $] 3&&y1Y> >>&&y1Y> >>  I}k R7rnc t|Dcgc]\}}t||s|c}}}g}|D]<\}}t||r||vr|j||f*|j||f>|Scc}}wrw)rrDrGr)rkpossible_fusionsdeferred_prologue_fusionsn1n2epilogue_template_nodesnew_possible_fusionss rl_handle_template_overlapz"Scheduler._handle_template_overlapUs#-. M.FB2DR2LR. M#  "&FB!"b)b4K.K)00"b:$++RH5 ' $# Ns A1 A1 c|j|t|}tjtj r@tj d|D]&}tj d|j(i}i}g}|j||}tjstjr|j||}|j||||||j|||j|||j!|r'|j||||||j||t#|d}|j%|}|S)a Combine eligible nodes into FusedSchedulerNodes. This relies on two key functions to control the logic: - self.can_fuse(): checks if a fusion is legal - self.score_fusion(): assigns priority to a given fusion zfuse_nodes_once, candidates:z %sc|jSrwrrs rlrvz+Scheduler.fuse_nodes_once..s!++rnr$)rrrrrrrrget_possible_fusionsr(rrr<r0r4r$clearr)r{) rkrrlrrr*r-r7r6s rlrmzScheduler.fuse_nodes_onceisd !!%( '  " "7== 1   ; <#  )=)=)?@$   OQ  " 44     # #v':':#<< ";      !     $$[/B //0E{S##% $  " ")%    3 34I; W{(=>..u5 rncht|j}d}t|j}tj d|t t j|D]P\}}t j|}t|dkr+|||kDrn|j|stj d|]|dz }tjdkD}t |dj|d|}tjd t|||D]} |j| |j||j j#|j%D cic]} | j'|c} |j(j+|d} | B| |j(|<St-|d |_|j/|j|_tjd ||t|j|j1|jycc} w) z' Groups parallel nodes rz2ComboKernels: Generating with num_ck_nodes = %s...rNz)ComboKernels: Not speeding up %d-th grouprTrz0ComboKernels: Combining %d nodes for %d-th groupc|jSrwrrs rlrvz5Scheduler.create_combo_kernel_nodes..sq{{rnr$zDGenerated ComboKernel nodes: %d ComboKernels, totally %d -> %d nodes)rrrrrrrorrspeedup_by_combo_kernelr(rrrr\rrrrrrrr)r{r) rkrXrrnnum_nodes_orignumrrrrrstreams rlrz#Scheduler.create_combo_kernel_nodess!, TZZ F U' & D DT J NC3CCINI9~!'EL,@// : EsK QJE$;;a?O4! &&*. / K HHBI  """4(" OOK (  # # * *4?4I4I4KL4Kq{*4KL  ((,,Yq\:F!39##K0E FK-BC 33DJJ?   R    O  !!$**-Ms"H/ cH|D]}|j|jyrw)rr)rkrrs rlrzScheduler.prune_redundant_depss D  % %d&=&= >rnc g ttttf d fd }tjt }|D]=}j |r|jD]}||j|?|jD] }|| tjratjt }|D]&}t|dd} | s|| j|(|jD] }|| j  jjdt j#dt%  S)z^ Helper to find all legal fusion opportunities, sorted by self.score_fusion() c|t|D]\}}||dz|dztjzD]}||f}|vr j|j ||rj |B|j s|jscj ||swj ||fyr[)rr()max_fusion_buffer_group_pairwise_attemptsrrrr r$) r node1_indexrrr%rlr6rIrks rlcheck_all_pairsz7Scheduler.get_possible_fusions..check_all_pairss&/&6" U"!Ok'FF'GE !%.Cd{ HHSM}}UE3CD(//4++-1A1A1Cu&6J)//?!'7rnrNT)r%reversezfound %d possible fusionsrrrrU)rrr_rrrunfusable_noderrrr(aggressive_fusionr*get_possible_fusions_with_highest_priorityrscore_fusion_keyrrr) rkrrlrLbuffer_names_groupingrr node_groupinggroup_groupingrr6rIs ` ` @@rlr?zScheduler.get_possible_fusionssU% 13D DEFH @ @(!, 7 7 =D""4(--/%c*11$70 399;M M *<  # #(44T:Ngt4"5)006"0!6!6!8  ."9 JJ   $"7"7F4c:J6KLrncttdfd |jjj |jjj z|j jj |j jj zz t fdD}|rt||d|S)z~ Finds whether there's a path from node1 to node2 (or vice-versa) caused indirectly by other fusions. ct|trq|vrmj||jj ryt |j zxs"tfd|j z DSy)NFc3HK|]}j|ywrwrSrr found_pathrks rlrzIScheduler.will_fusion_create_cycle..found_path..:s*H!DA#4#:#:1#=>!D")rrrrissubsetrvrr)rcombined_ancestorscombined_namesrZrkvisiteds rlrZz6Scheduler.will_fusion_create_cycle..found_path)s$ 23G8K D!++-667IJ!  ?@CH!%2D!DHErnc3HK|]}j|ywrwrSrYs rlrz5Scheduler.will_fusion_create_cycle..Hs$WDVqJt66q9:DVr[zwill create cycler )rrr_dictrrrr)rkrrcycler]r^rZr_s` @@@@rlrz"Scheduler.will_fusion_create_cycles/02  2  % % ' - - 2 2 4'')//446 7  OO ! ! & & (5??+@+@+E+E+G G WDVWW  #IeU #$7 8 rnc ddlm  d fd }||}||}t fd|D}t fd|D}|j|}d} |D]} | t | dz } j ||} tjjj| d | zry y#t $rYywxYw) a Return true if fusing the two nodes can potentially increasing peak memory. The implementation is more like a heuristic since we don't really know if we are at peak or not when trying to fuse these two nodes. The order of nodes may change later which makes the peak memory estimation hard. Here is how we decide the LOWER BOUND of extra memory allocation if we fuse these 2 nodes: 1. find all buffers read by each node with a single user. These buffers are supposed to be reused if we don't fuses these 2 nodes 2. find the intersection of these buffers for the two node and sum the total buffer size. If we don't fuse these two nodes, we can at lease avoid this much memory allocation. Note that the extra memory allocation is not necessarily causing peak memory increase. This is just a heuristic. We return true only if the saving for fusion can not trade off the extra memory allocation. r)buffer_reuse_keyc0g}|jjD]y}jj|j}|s+t |j dk(sD|jjs_|j|j{|Sr[) rrr@rrrrrhas_tensor_outputr)rr{rxrrks rl_find_single_user_inputszKScheduler.can_fusion_increase_peak_memory.._find_single_user_inputsdsrF&&,,&&**27733syy>Q.3883M3M3OMM#((+-Mrnc3.K|] }|ywrwr~rrrds rlrz.r#S]c$4S$9]r>c3.K|] }|ywrwr~ris rlrz.srjr>rrFrT)rr_rzlist[ir.Buffer]) r7rdr intersectionr rrIrZrrstatically_known_gt) rkrrrg lhs_dep_nodes rhs_dep_nodeslhs_reuse_keysrhs_reuse_keyscommon_reuse_keysmemory_overheadr% bw_savingrds ` @rlcan_fusion_increase_peak_memoryz)Scheduler.can_fusion_increase_peak_memoryMs* 6 #   17 07 ##S]#SS##S]#SS*77G$C 3s1v;.%,,UE:  77   / /i P  s$B88 CCct|jDcgc]}|jc}|jDcgc]}|jc}z}td|jjD}td|jj D}||z}t} |jjD]:} |j | j|s | j| j<td|jj Dtd|jj Dz} td|jjDtd|jjDz} | |z } | | z }| |z}t||kDScc}wcc}w)Nc34K|]}|jywrwrrs rlrzEScheduler.fusion_prevent_too_many_reads_and_writes..s&T;SCsxx;Src34K|]}|jywrwrrs rlrzEScheduler.fusion_prevent_too_many_reads_and_writes..%R:Q3chh:Qrc34K|]}|jywrwrrs rlrzEScheduler.fusion_prevent_too_many_reads_and_writes..s$ 7CHH 7rc34K|]}|jywrwrrs rlrzEScheduler.fusion_prevent_too_many_reads_and_writes..ryrc34K|]}|jywrwrrs rlrzEScheduler.fusion_prevent_too_many_reads_and_writes..s% 8CHH 8rc34K|]}|jywrwrrs rlrzEScheduler.fusion_prevent_too_many_reads_and_writes..sD+CCsxx+Cr) rrrrrr$can_buffer_be_removed_through_fusionrrr)rkrr thresholdrfused_node_namesnode1_write_namesnode2_read_namesreads_removed_through_fusionwrites_removed_through_fusionrall_read_namesall_write_names unique_reads unique_writesunique_io_bufferss rl(fusion_prevent_too_many_reads_and_writesz2Scheduler.fusion_prevent_too_many_reads_and_writess&).): ;):T]]_): ;+0??+<=+<4t}}+<= > '&T5;L;L;S;S&TT%%R%:K:K:Q:Q%RR'7:K'K$:D%**11I88 0.11)..A 2$$ % 1 1 7 7$  C5+<+<+B+BC CD %% % 1 1 8 8%  D5+<+<+C+CD DE &(DD (*GG )=8$% 11M <=s GG ctt|j|jz t|j|jz }|dkDS)aA This function prevents fusion for nodes that can increase memory footprint. This problem is more common in horizontal fusion, where nodes that are far apart in the original order get fused, lengthening the live intervals of tensors. This is very evident in models with activation checkpointing, where the recomputed nodes from different checkpointed regions get fused and significantly increase the memory footprint. The current attempt is a quick, possibly hacky, heuristic to prevent the fusion of nodes that are far away in the original order. A better but difficult to implement heuristic would be to use live intervals of the buffers, find region of peak pressure in the original program and prevent fusion that crosses that peak region. We might need special care or good approximation in this implementation, as fusion of node changes live intervals, and re-computing live intervals and peak memory after each fusion can introduce large compilation overhead. @)rrrcrd)rkrrproximity_scores rlare_long_distant_nodesz Scheduler.are_long_distant_nodessE* %//1 2 %//1 2 ##rnci}|jjDcic]}|j|}}|jjDcic]}|j|}}|D]}}tjj |} ||} ||} t | trt | tsdt| dt| ||<k| j| jk7r(d| jd| j||<t| jt| jk7rd||<| j} | j} | | k7r d| d| ||<| j| jk(r d| d| ||<Ed}t | tjsd| j }d | d| d |||<t#|Scc}wcc}w) z} Try to decide reasons why fusion fail due to no shared memory even though there are common buffers. znot MemoryDep: rzdifferent numel: broadcastzdifferent offset: zMismatch loop orders: rzLayout: zUnknown reason: z. )rrrrZrrrr4rrrYr get_offsetnormalize_with_stride_orderr+rr'r )rkrrcommon_buf_namesreasonsrnode1_name2depnode2_name2deprrlhs_deprhs_deplhs_offrhs_off layout_strs rldecide_fusion_fail_reasonz#Scheduler.decide_fusion_fail_reasons383D3D3U3U3WX3WC#((C-3WX383D3D3U3U3WX3WC#((C-3WX(H''$$X.C$X.G$X.Ggy1GY9W%d7m_F4=/J!  "g&7&7&99'(9(9(;'j@jC|<tE|}|||d }|y|j&j(||j&j(|<||j&j(|<|jGd d |jI||}t|tJsJtLjOd ||Scc} w)aW Attempts to enable fusion between two nodes by inverting indexing patterns. This optimization targets cases where node1 has a contiguous write and node2 has a contiguous write but discontiguous read. By inverting the indexing in node2's read and write operations, we can make them compatible with node1 for potential fusion. Args: node1: First scheduler node (source) node2: Second scheduler node (target for inversion) Returns: int: Fusion score if successful, 0 if optimization not applicable rc3<K|]}|jywrwrrs rlrzAScheduler.shared_data_after_inverting_indexing..%s2>aqxxz>rc34K|]}|jywrwrrs rlrzAScheduler.shared_data_after_inverting_indexing..1s. 8CHH 8rrrindex0index1c3 K|]}|ywrwr~)rrs rlrzAScheduler.shared_data_after_inverting_indexing..ls%T7Std7Ss r)generate_inverse_formulaTFz!Shared memory after inversion: %d)(r(rnrr buffer_namesrrqrrrrrrr4rr`rr var_namesrr subblocksget_read_exprs$torch._inductor.invert_expr_analysisrvarsrAdd make_argsrrZrrcombine_modular_indexing_pairsrrzrIr rr)rkrrnode1_buffer_namesnode2_buffer_namescommon_buffer_namesnode2_unmet_dependencies node2_read node2_writer node1_writes node1_writenode2_read_exprs read_exprread_expr_indexwrite_expr_indexrrsimplified_termstermsimplified_read_exprinverse_formulascores rl$shared_data_after_inverting_indexingz.Scheduler.shared_data_after_inverting_indexings&44 2E5>2 2#..;;="..;;=03EE"$.. % 8 8. $   $&8 8 ' (1 , u  && '! +s53D3D3K3K/Lq/P$u006678 4 1 1 8 89: *i0 9 161B1B1I1IJ1I## 1I J ??, .":??3 +y1 "++-    !2!2 2  K$4$44 ??k.. .#j6J6J2Kq2P u{{)) *a / ;;   22 2EKK666  7 &%Tu{{7Q7Q7S%TT  A %./0   228< <&O'  : :8 DD DD&O' Q[[%%a( z?a II'' 2D  # #  ??E 3 ##3423GTUW  " 7|jA||SdScc}wcc}w)a Right now just greedily reorder the loop of node1 to be compatible with node2, but ideally we should have some heuristics to reorder the loop for node2 to be compatible with node1 if that's more efficient. Return the amount of shared data re-computed in this method. If no such recomputation happens, return -1 (not return 0 since 0 is a valid amount of shared data). c3<K|]}|jywrwrrs rlrz>Scheduler.shared_data_after_reordering_loop..s8 .1AHHJrrrrr$Fz?Don't reorder loops since both nodes are reductions: %s v.s. %s)!r(rerr rrrrrrrZrrrrrrrrrr4rr` dep_size_hintrrrrrrGrHr rI)rkrrrrrrrr candidatesrrr_numel reordereds rl!shared_data_after_reordering_loopz+Scheduler.shared_data_after_reordering_loops 00C8 !&8 5     %"3"3"5"..;;="..;;=03EE"383D3D3U3U3WX3WC#((C-3WX383D3D3U3U3WX3WC#((C-3WX .K$[1G$[1G3356689!!((::#--/!;  /" z?a $'zx7J7J17M#N '9-Z5S   w// /   "g&7&7&99))'22 !!#77II##%77II  # #Q    KKT55eUC D  kYXs J0J5cft|tr)|j xrt|j St|t rht|jt jr|jj S|j xrt|j Sy)z> Is this node unfusable under any conditions. F) rrAr rWrrr+r@rAr%s rlrOzScheduler.unfusable_nodes d2 3''))2U 3/  d5 6$))R%?%?@9966888''))2U 3/ rnc|jtjjkry|j }|j }d}|||zkDr |dyt d|jD}|tjjjjfk(r |dyd d}|j} | js+|| jr|j!s |dyy) zT Heuristics to avoid benchmarking predictably slow prologue fusions T皙?z@prologue fusion will not increase amount of bytes read in kernelFc3K|]J}|j<|jjD]}|jdk(r|j!Lyw)N call_function)rrWrrX)rrrs rlrzEScheduler.check_prologue_fusion_heuristics_fusable..+sN .vv!VV'')tt& HH* .sAAz\prologue fusion will not increase attempt to fuse in padding bc it increases unaligned readsc<|jdkxr |jS)Nr)itemsizeis_floating_point)rs rl low_prec_fpzGScheduler.check_prologue_fusion_heuristics_fusable..low_prec_fp8s>>Q&B5+B+B BrnzVprologue fusion that must be upcast to fp32 not profitable for low precision templates)rz torch.dtyperrv)rrZrinvoke_quant_opsrlrnrrrrrconstant_pad_ndr*rrVrr ) rk prologue_noderr read_bytes write_bytesBYTES_THRESHOLD_MULTIPLIERr_r template_bufs rl(check_prologue_fusion_heuristics_fusablez2Scheduler.check_prologue_fusion_heuristics_fusable s  , , .!''2J2J J"88: #::< &)" 'AA B R S ",,.   uyy~~55==? ? n  C%??A 668L../!>>@ h rnc<t|trt|tsyt|jtjr$t|jtjsy|j s|j ryt jdk(ry|j|j}}|\}}|\}}|js,|js||k7st|t|k7ryt|jjdkDs"t|jjdkDryjtt|jj} jtt|jj} t!| | t j"kDrydfd } | |s| |ryg} t%t'||D]\} \}}||k7s| j)| !t| dk7ry| d}||||}}t*j,j.j1||r|||fSt*j,j.j1||r|||fSy)ao Fusing two small pointwise nodes significantly reduces kernel overhead and launch overhead. However, slightly different sizes would prevent fusion. Here, we decide if expanding sizes of one node is profitible by allowing fusion, and returns the dimension to expand, node with smaller sizes, and new size after expand. Nrirc~|jjD]}|jjvrj|j}n%jj |j}|s]t jjj||st|jtryyr) rrrr?r@rrZrr:r@rrrA)rrHrIrks rlhas_reusable_bufferzIScheduler.get_expand_dim_for_pointwise_nodes..has_reusable_buffers((..99 ; ;; $ ; ;DII FI $ 0 0 4 4TYY ?I,,66y$G&y'<'<>TU/rnrr )rrrr+rrr(rjrYrrrrrrrrsmall_memory_access_thresholdrrrrZrrstatically_known_lt)rkrrn1_sizesn2_sizes n1_iter_sizesn1_reduce_sizes n2_iter_sizesn2_reduce_sizesnode1_write_memorynode2_write_memoryrmismatch_dimensionsidxn1_sizen2_size mismatch_dimmismatch_size1mismatch_size2s` rl"get_expand_dim_for_pointwise_nodesz,Scheduler.get_expand_dim_for_pointwise_nodesHsY%/z%7W uzz2#4#4 55::r'8'89  ) ) +u/M/M/O    )#\\5<<()1& )1&     !!#/1=!S%77 u  '' (1 ,E4E4E4L4L0MPQ0Q"//T%:K:K:R:R5S0TU!//T%:K:K:R:R5S0TU "$6 722 3  u %)jj>k7r |dy d%fd t3fd|j@Drytttfrjs |dyjCjDzr |dyjrtGs |dyjIsjr |dyjK} | jM} | s |dytOd| jPD| z } jS| zr |dyjUsjUr |dyjWddD]B}|jY}|D]+}t[fd|j\Dr"|dyDtt^sgn*j`Dcgc]}|js|c}}t-|d k(sJ|d }t-djbd k(rSt-djbd j\d k(r+djbd j\d j|us |dy|je|syjrjUsjIs tgs |dyji}|J|jkr-tjtj&s |d yjStljnjpzs+jStljnjpzr |d!yj}j}||k7r |d"||y~|js|#}t|ttsJ|r<|tvjxkr)tvjzr|j}}|d k\r|}tvj~rP|jx}r<|\}}}|j|||js}t|ttsJtvjr,|tvjxkr|j}|d k\r|}tjtjr4tjd$jj|tljj||syjCjDzrY|jxrEtljj||xr!|j|jStljj||xr!|j|jScc}w)&zj Determine if it is possible to combine node1 and node2 into a single fused node. FNTz/grouped node must not be fused with other nodesz node1 is nopz'node1 is extern but not a triton kernelz5node1's triton kernel doesn't support epilogue fusionz.node1 is extern but node2 is not SchedulerNodez3node1 is extern but node2.node is not SchedulerNodez4node1 is extern but node2.node.data is not Pointwiserrc3<K|]}|jk7ywrwr)rrwritten_buffer_names rlrz%Scheduler.can_fuse..sV>Us38822>UrTz9epilogue reads from buffers other than the mutated outputc3&K|] }|dk7 yw)rNr~)rusages rlrz%Scheduler.can_fuse..s;F5uFsz*node1 and node2 uses different buf layoutsc@|uxr|uxr|jvSrw)r)rrrrs rl._is_other_node_that_references_mutation_bufferzJScheduler.can_fuse.._is_other_node_that_references_mutation_buffers7 u,N#50N+z/K/K/MMrnc3.K|] }|ywrwr~)rrrs rlrz%Scheduler.can_fuse.. s&D?tD&r>znode2 is extern or nopznode1 must go before node2zprologue fusion turned offz2prologue fusion only supported for pointwise nodesz'template has no allowed prologue inputsc3<K|]}|jywrwr)rrs rlrz%Scheduler.can_fuse..*sE_c3<<>_rz;prologue fusion not implemented for kernel for these inputsz:template prologue can only fuse functional pointwise nodesrc3:K|]}|jvywrwr)rr.prologue_nodess rlrz%Scheduler.can_fuse..:sQyttyyN:yrz7template prologue can only fuse nodes with a single usezEtemplate prologue can only fuse nodes with a single use into templateztemplate epilogue not satisfiedz6multi-output template epilogue requires ComputedBufferz#fusion for buffer explicit disabledzdevice mismatch (%s vs %s)r;z%s and %s has %s shared data)rr_)Nrrrrr8rOrr rcrcan_fuse_multi_outputs_templaterrArrr+r@rArrrrrmutation_outputsrrrrinner_fn_free_symbolscollect_inner_fn_symbol_usagerCr'rrrrBrrget_allowed_prologue_inpsrrrrrrzrrrrrjrr=rrVrZrno_fuse_buffer_namesrIr r(score_fusion_memory_thresholdrer$expand_dimension_for_pointwise_nodesrrrnrrrrrrrrrcan_fuse_verticalcan_fuse_horizontal) rkrr can_reorderr<rstream2rnode2_inner_fn_free_symbolssymbolusagesrrunsupported_prologue_argsr node_outsrrtemplate_snodestemplate_snoderrdevice2shared_data_scorenew_shared_data_scoreexpand_analysis expand_dim smaller_node expand_sizerrrs `` @@@rlrzScheduler.can_fusesG E>  ' ' )))--e4G))--e4G"w':w'?Q e4 5&&u- - e4 5u%    4#3#3    $ ) )% 7$8 e1 2j '7  A B e3 4U=N=N=P   e6 7ejj"*D*DE=>:://1KLe]3DEejj.9IJejjooy9JKuzz223q8 88"'**"="=a"@"E"E Ve>O>O>U>UVVOP+0**//*O*O*Q '5FFvN;F;; 6 uzz../14 44zz&&q)00EJJ4E4EE@A -  JJ u8:PQ R%%' ( )  $ $ & 8 , -    .u501!!#u'8'8':HI779H$,$F$F$H !(=>EX__EE'( & %%'*CCQR--/53Q3Q3SPQ"__.N&s+ ,,. $CQsyyQQUV$%,"%);<!&AAaA  '1, ,,,Q/NN2&../14r*2215;;<A"2&..q177:??>Q[@@sS    ..0%%'25956 224L+ ++557  B--ALM  " " $qww'C'C C  " " $qww'C'C C 5 6!!#""$ W  ,fg >  44 54M5 +S111 !F$H$HH11$($J$J5RW$X !$)$9!  6 6#FFueT TO T6E 3Z{  < >ueL  9900eU$5M""6*>>ueL MCBs d%$d%c|j}t||}tt}|jD]j}|j j |j|j}t|tr|j|||rW||j|l|jjD]}t|tst|ts$|j |j j |j|j} | sf| D]v} t|tr$|j!| |r| j#| 7t|tsH|j%| ||j&sf| j#| xt)dt*j,j/|j1D} | |zr |dy|j3} | D]E}|j4|j7} | |j8| j:zs=|dyy)a Check if it is legal to fuse a consumer (node2) into a producer (node1). We can fuse them if all the reads of node2 either match corresponding writes in node1, or are written by nodes that can be scheduled before the fusion of node1 and node2. c34K|]}|jywrwrrs rlrz.Scheduler.can_fuse_vertical..s$ U HHUrzmemory deps did not matchFz(intermediate nodes between node1 & node2T)rrrrrqrgrrrr6r5rrrr4r5fusable_read_and_writer\.fusable_stardep_write_and_read_on_empty_tensorrrrrrrrr@rrr)rkrrnode1_buf_namesrremaining_deps_by_namerrcd remainingrxremaining_depsnode1_op_namesrs rlrzScheduler.can_fuse_verticals 002u%7B47H++C((,,SXXsxx@D#w'D,A,A#ue,T "4 ( / / 4 , ##**Bb),ZG5L.22%%))"''277;I#B!"i0T5P5PB6"((,#GMMB "((,$+&$$  445K5R5R5TU$   O + + ,224"D&&t,==?G 7 7 @ J JJ>? # rnc|j|jvry|jjDcgc]}|j|jk(r|}}t |dk7ry|dt tryt tsJtjtjrytjjjksy|j |j}|g}t |t"r |j$}d}|D]R} | jj&D cgc]} | j|k(r| } } | s8|dz }t)fd| DrRy|dkScc}wcc} w)NFrrc3K|]q}t|txr[t|jtj  xr4|jjk(xr|j j k(sywrw)rr4r#rr%TMPr)rrHwrites rlrz-Scheduler.fusable_weak_dep..so +D 4+,+DJJAA,JJ%++-,II+,+sA7A:)rrrrrrrr5r4r#rr%rrrr"rrorrr) rkweak_deprrrmutating_writesrrelevant_reading_nodesnum_concurrent_reads reading_noderHrelevant_readss ` rlr5zScheduler.fusable_weak_deps == 6 6 8 8 **11 1zzX222 1   1 $" eW %%+++ u{{DHH 5 %//*ekk.F.FF++H,A,AB "' e7 8%*\\ " 2L)44:::D99 ):  " A %  + !3"$q((W 6s "E:;E?c$t|trd|jj|j|j}||jk7sHt |j tjs$t |j tjrytjr9|j|jk7r |j}|j}|j|jry|j |j k(xr\t|j t|j k\xr/|j dt|j |j k(St|t"r|jj|j|j}|jj|j|j}|j|jk(r|j||k(ryyrF)rr4rgrrr#rr%rr(rerr`rDrrrr5)rkrHr read_name write_names rlrz Scheduler.fusable_read_and_writesw dI &--11$))TYYGIUZZ'&tzz488<&u{{DHH=00T]]enn5T~~')11%**= ekk)? Nc%**o5?II/EJJ0EJJ>  g &--11$))TYYGI..225::uzzJJ UZZ'JJ*+rncHt|tjsy|jsy|jj |j |j }|jj |j |j }t|tr||k(ryyrF)rr+r@rArgrrr5)rkrHr writing_noder'r(s rlrz8Scheduler.fusable_stardep_write_and_read_on_empty_tensorFs},(B(BC--/))--diiC **..uzz5::F eW %)z*ArncBtjj||Srw)rZrget_dep_size_hint)rkrrEs rlrzScheduler.dep_size_hintSsww((k::rnc Bfd}|r6tj||r tj||}||ddSt|jt j r|jjs |js|jr|jj|jjz}|jj|jjz} d d} d}|D]@} | D]9} | | | s |tj| j| z };B||ddSt|jjt|jjz} t|jjt|jjz}t| |dzt| |kr| |kDr||}}|jj|jjzDcgc]4}||jjvs||jjvr|6}}|t!fd|DddS|jj|jjz|jj|jjzz}t!fd|D}d}|dk(r$j#||rj%||}|||dScc}w) a4 The first term in our fusion score that estimates number of saved memory operations. This function scores fusion candidates based on shared memory access patterns. Higher scores indicate better fusion candidates. Scoring strategy: 1. If nodes share exact memory deps (same buffer + same indexing), return the sum of shared dep sizes (original behavior). 2. If no exact matches (score == 0), check for same-buffer reads with different indexing (e.g., split operations reading different slices). - Give bonus if nodes read from exactly the same set of buffers - Score based on overlap ratio: common_buffer_size / total_read_size - High overlap (>50%) suggests good cache locality benefit from fusion cr|||fS||zSrwr~)rbuffer_overlap_scoreis_mix_order_reductionreturn_is_mix_order_reductions rl_construct_return_valuez>Scheduler.score_fusion_memory.._construct_return_valueos#-35KLL// /rnrTc||k(ryt|ttfr/t|ttfr|j|jk(Syr)rr5r4r)dep1dep2s rl_matchz-Scheduler.score_fusion_memory.._matchsD4<dWi$89j7I.? 99 11rnFr&c3BK|]}j|ywrwr)rrrErks rlrz0Scheduler.score_fusion_memory..sIDSD&&sK8Drc3@K|]}j|ywrwr8rs rlrz0Scheduler.score_fusion_memory..sJ7ID&&s+7Ir1)r4r3r5r3)rrrrrr+r@rAr rrrrrrrr_can_use_buffer_overlap_scoring&_score_fusion_memory_by_buffer_overlap)rkrrrEr1r<r2r node1_deps node2_depsr6 node1_dep node2_dep node1_dep_len node2_dep_lenrrcommon_memory_depsr/s` `` rlrIzScheduler.score_fusion_memoryVs2 0 %):)C)CE5)Q &66ueDE*5!T: :5::r'A'ABJJ002  "  "**0053D3D3K3KKJ**0053D3D3K3KKJ E' !+Ii3 ..y94;M;Mi;X"",(+5!U; ;E--334s5;L;L;S;S7TT E--334s5;L;L;S;S7TT  }m ,q 03}m3T T},$eu!,,22U5F5F5M5MMMC%++111SE&> > J7IJJ ! A:$>>ueL#'#N#Nu$ 'u.BEJJ3s 9Lct|js|jry|js|jrytjstjrU|j }|j }|r|syt d|D}t d|D}t }|D]}|jD]} t| jts| jjs9t| jsO| jj} | Lt| tjr2| j} || zs|j!| j|j!| j|r|D]}|jD]} t| jts| jjs9| j|vsH| jj} | 3t| tjr| j} || zsyy||fD]d} | j"j$D]I} |j&j)| j*}|+|js.+TmsCLLNmrc3<K|]}|jywrwrrs rlrz.rEr)rr r(rrrzrrrrr_rBrr+rrrrrrrrr=)rkrr node1_outputs node2_outputsnode1_output_namesnode2_output_names&node1_prologue_eligible_template_usersrr.r allowed_inpsrrrrs rlr:z)Scheduler._can_use_buffer_overlap_scoringsD4    5#5#5#7    %"3"3"5   &":":!--/M!--/M! !++Tm+T!T !++Tm+T!T    3%IID"499.?@ II1137 B )- (C(C(E (4)2+B+B:,9+R+R+TL1L@ F J J499 UCFFtyyQ%&%.6(C # &tyy2CD $ 5 5 7 $ -S S-1II,G,G,IM,8Z -r/F/F>0=/V/V/X #5 #D+0(-!!*)&++11C#66::388DH ,$0027A$2'rnc| d d fd td|jjD}td|jjD}||z sytfd|jjD}tfd|jjD}t ||}|dk(ryt fd|jjD}t fd |jjD} t || } | |z } | t j k\r| SdS) a8 Score fusion based on buffer name overlap when exact dep matching fails. This handles the split/cat fusion case where nodes read from the same buffer but at different indices (e.g., different slices from a split operation). Scoring logic: - If nodes read from exactly the same buffers: high bonus (encourages fusion) - For common buffers: score based on overlap ratio - overlap_ratio = common_buffer_size / max(node1_total_reads, node2_total_reads) - If overlap_ratio > threshold (e.g., 0.5): give proportional score - If overlap_ratio < threshold: minimal/no score (not worth fusing) Note on dynamic shapes: - When deps have unbacked symbols (dynamic shapes), dep_size_hint returns 0 - In this case, we use count * 10 as a proxy for size - This ensures fusion still works for models with dynamic batch sizes Note on multiple deps from same buffer: - A node may have multiple MemoryDep entries for the same buffer name (e.g., 4 split reads from arg0_1 at different indices) - We sum ALL dep sizes for each buffer, not just take max - This ensures overlap ratio is calculated correctly when nodes read multiple slices from the same underlying buffer r c8j|}|dkDr|SSrr8)rrFALLBACK_DEP_SIZErks rl get_dep_sizezFScheduler._score_fusion_memory_by_buffer_overlap..get_dep_sizeKs%%%c*D!84 :): :rnc34K|]}|jywrwrrs rlrzCScheduler._score_fusion_memory_by_buffer_overlap..Oryrc34K|]}|jywrwrrs rlrzCScheduler._score_fusion_memory_by_buffer_overlap..Pryrrc3.K|] }|ywrwr~rrrPs rlrzCScheduler._score_fusion_memory_by_buffer_overlap..Y$ )@#L )@r>c3.K|] }|ywrwr~rTs rlrzCScheduler._score_fusion_memory_by_buffer_overlap..\rUr>c3JK|]}|jvr |ywrwrrr common_namesrPs rlrzCScheduler._score_fusion_memory_by_buffer_overlap..g*% .xx<'  . #c3JK|]}|jvr |ywrwrrXs rlrzCScheduler._score_fusion_memory_by_buffer_overlap..lrZr[)rr3rr )rrrrrr(min_overlap_ratio)rkrrnode1_read_namesrnode1_total_read_sizenode2_total_read_sizemax_total_read_sizenode1_common_read_sizenode2_common_read_sizecommon_read_buffer_size overlap_ratiorOrYrPs` @@@rlr;z0Scheduler._score_fusion_memory_by_buffer_overlap+sQ< ;&%R%:K:K:Q:Q%RR%%R%:K:K:Q:Q%RR(*:: !$$ ).):):)@)@$ ! !$$ ).):):)@)@$ ! ""79NO ! # "%% ((..% "  "%% ((..% " #&&<>T"U02EE (58P8P'P # VW rnct|dk(r|Si}|D]\}}|j|jk(sJ|j}t|j|j ||}||vr ||fg||<p||j ||ft |jtjdd}t|dkDsJ|S)Nrr$r) rrr rcget_fusion_pair_priorityrrrrr)rkr6"possible_fusions_group_by_priorityrrrfusion_pair_priority&possible_fusions_with_highest_prioritys rlrQz4Scheduler.get_possible_fusions_with_highest_prioritys  A %# #  +-LE5##%)9)9);; ;;%%'F#&  (AA%O$ $+MMENL23GH33GHOOEN-25 . 4 4 6H>>55rncBtjj|g|S)z- Shim for list.sort(key=...) )rZr score_fusionrOs rlrRzScheduler.score_fusion_keys yy%%d3U33rncttjj}t |j D]9}|j ||j|j|j;y)zg Populate node.last_usage recursively (also for the nodes within a FusedSchedulerNode) N) rrZrrrrrrrr`)rkrrs rlrzScheduler.compute_last_usagesX ))A)A)CDTZZ(D    3T5L5L M  & &t 7)rnct|jtjjz tjj j z D]z}||jvrT|j|}|js2tjj j|jf|tjjvstjj|}t|tjr*tjj j|t|tjtj fr|j"}t|tj$r|j'sJtjj j|j"}|jj)y)z*Free any buffers that are no longer neededN)r)rrZrr?r:freedr@rI codegen_freerrrr+rrrrris_input_bufferr@)rkrrrstorages rl free_bufferszScheduler.free_bufferssO  % %gg%% &gg""(( ) D t'''&&t,<<>GG((55chh?---gg**40c2#5#56GG((55c:b&7&79M9M%NO!hhG"7BMM:w?V?V?XXGG((55gllC) , !!'')rnc|jjD]}|j|jyrw)rlrflushrs)rkrQs rlruzScheduler.flushs.}}++-G MMO. rncvt|ttfsJtddxxdz cc<t j t d5|j|jddd|jtjj|jy#1swYCxYw)Nr extern_callsrF)increase_kernel_count) rrrVrrZset_kernel_handlerr0rLrr;rr:rs)rkscheduler_nodes rlcodegen_extern_callzScheduler.codegen_extern_calls  &(L M    ^,1,  ! !&u"E F  0 0 2  # # %G qww334  G Fs !B//B8cPt|jr|j J|dtjj |t |j}|td|jts|jdk(rLtjj|x}jdkrt|tjt|jr,|jdk(st!tj||S)Nz( should have been normalized in loweringzUnsupported device type: rr)rUrrrZradd_device_infor/rUr&rrget_device_propertiesmajorr7inspect currentframer8)rkrdevice_schedulingrs rlcreate_backendzScheduler.create_backends&++&&,,*B h> ? B '5fkkB  $!:6;;-HI I| v%%*ZZ%E%Ef%MM\TTWXX(w7K7K7MNN $V[[E-A#G$8$8$:;; &&rnc|J||jvr|j||j|<|j|Srw)rlrrs rlrczScheduler.get_backendsB!!!  &$($7$7$?DMM& !}}V$$rncdfd }|jDcic]8}|j*|jjD] }|||fd:}}}t|j }|rMt |t jd\}}tjjj|yycc}}w)Nc|jvrLjjt|jjDcic]\}}|| c}}jScc}}wrw)rrrrr)rrrks rl get_orderz*Scheduler.enter_context..get_ordersb,,,$$++i >V,W>VdaQT>V,WX''* *-Xs A+ rr$)rz torch.fx.Noderr ) rrrWrrrrrrZrr: enter_context)rkrrrrr_rlasts` rlrzScheduler.enter_contexts +^^% %vv!VV'')q\1 t #* %  w||~& 'x':':1'=>GAt GG . .t 4  s=Cc |j|j}tfd|Dxr||jvxr||j vS#t$rYywxYw)NFc3^K|]$}|jxs|jv&ywrw)rr)rr.rs rlrzAScheduler.can_buffer_be_removed_through_fusion..s,VPU C 3C CCPUs*-)r@rKeyErrorrrgr)rkrrrs ` rlr~z.Scheduler.can_buffer_be_removed_through_fusionsn $$T*00E VPUV V 4D111 4D333   sA AAc|j}t|tjjj rk|j x}r]t|\}}|tjvs|tjvr+t|tjjsJd|Stjjjjstjyt|t r)|j"D]}|j%|}|s|cSy|jJ|j's|j)dSt|jtj*ryt|jtj,ryt/|jddryt1|jry |j3|x}r|Stjj4r t7|ry y) z Return the reason why we should partition the inductor graph on this node, or None if the node is cudagraphable. zcustom partition op: Nz6partition includes all ops when cudagraphs is disabledz opszDeviceCopy opszConditional opsunbacked_bindingszunbacked binding opszCUDAGraph-unsafe custom opszdynamic shape ops)rrrrr+rDrrPr(custom_should_partition_ops_ops OpOverloadrr^rHrNrrshould_partitionrUr DeviceCopyrLrrT&_uses_cudagraph_unsafe_unbacked_symintcudagraph_skip_dynamic_graphsr2)rkrr5rop_overload_packet_nameop_overload_namerrs rlrzScheduler.should_partitions)) gu11@@ A%% %B %8DR8H 5 #%5'6+M+MM#v'I'II!"ejj&;&;<<<./?.@AA &&--886>>FK d. /..u5!M%yy$$${{}oo'(- - dii /# dii 0$ 49914 8) !$)) ,0@@F F6 FM == 6 6-d3*rncTt}tjs|S|jD]}|j}|t |t jjjsA|j}|Pt|\}}|tjvr|tjvr|jD]g}tjjj!|}t#|t$j&t$j(fsW|j+|i|S)zc Collect output unbacked symints from ops in config.cudagraph_unsafe_unbacked_ops. )rr(cudagraph_unsafe_unbacked_opsrrrrrr+rDrrPrrZrrrr$r% UNBACKED_INTUNBACKED_FLOATr)rkunsafe_symintsrr5rrrsyms rl&_get_cudagraph_unsafe_unbacked_symintsz0Scheduler._get_cudagraph_unsafe_unbacked_symints]s 4><33! !JJDiiGgu'9'9'H'HI$$Bz8DR8H 5 #%5'v/S/SS$F,P,PP779gg&&//4!#(9(94;N;N'OP"&&s+:'0rnc|j}|syt|}|D]I}tjjj |}|j D]}||vsd|ccSKy)Nz'uses cudagraph-unsafe unbacked symint: )rr2rZrrrr")rkrr node_symbolsrsimplified_symfree_syms rlrz0Scheduler._uses_cudagraph_unsafe_unbacked_symintspDDF5d; CWW--66s;N*77~-DXJOO8 rnci}|jtjj|jD]3}|j j D]\}}|j||<5|S)z~ Return a mapping from name strings to the corresponding graph inputs or base scheduler node outputs. )rrZrrrrlrr)rkrurrscheduler_buffers rlget_name_to_nodeszScheduler.get_name_to_nodessiPR AGG001JJD*.*>*>*D*D*F&&%5%:%: T"+Grnc ttjjDcic]\}}|| }}}ttjj Dcic]\}}|| }}}gtj_t|D]\}}|j rg}|jD]"}|j|j|$g} |jD]0} | j|j| j2tjj jt||| |jycc}}wcc}}w)z computes a mapping from partition input/output indices to graph input/output indices for each partition. N)rrZrrrpartition_mapsskip_cudagraph input_nodesrr output_nodesrrQconstant_names) rk signaturesrrname_to_graph_input_indexname_to_graph_output_index partition_id signature input_mappingoutput_mappingrs rlcompute_graph_partition_mapsz&Scheduler.compute_graph_partition_mapssE(11E1E'F% 'F)#tD#I'F "% (11I1I1K'L& 'L)#tD#I'L #& "$'0'< #L)'' M!--$$%>%B%B4%HI. N!..%%&@&D&DT]]_&UV/ GG " " ) )! !",,  !(=% & s E! E c dd dd}tjd|D}|jfd|jD||}t}|D]F}tj j j|}|j|jHtt|tjdS) ai Returns all symbol inputs which are required to be in scope to successfully perform codegen for this graph partition, including: - free symbols used in partition nodes - free symbols in partition input/node shapes, strides, and offsets. This is needed for recording cudagraphs for tensors with dynamic shapes. ct|tjr tSt|tjr t |St dt|)zW Gets symbols used in input node shapes, strides, and offsets. zUnsupported input node type: )rr+rrrr-r+rrs rlget_input_node_symbolszKScheduler.get_graph_partition_symbol_inputs..get_input_node_symbolssN $ 2 23!|#D")),)$//*,I$t**VWWrnc&td|DS)z Filters a set of symbols that are required for codegen. Skip symbols that are always internal to kernels, such as SymT.TMP, SymT.INDEX, and SymT.R0_INDEX. c3K|]N}t|tjtjtjtj fr|Pywrw)r$r%SIZEFLOATrrrrts rlrzVScheduler.get_graph_partition_symbol_inputs..filter_symbols..sH  A!  ))++  sAAr)symbolss rlfilter_symbolszCScheduler.get_graph_partition_symbol_inputs..filter_symbolss     rnc32K|]}t|ywrwr1rs rlrz>Scheduler.get_graph_partition_symbol_inputs..s Iyt,T2yr3c34K|]\}}|ywrwr~)rrrrs rlrz>Scheduler.get_graph_partition_symbol_inputs..s N:Mwq$$T*:Msrr$)rz+ir.IRNode | sympy.Expr | ir.TorchBindObjectrOrderedSet[sympy.Symbol])rrrr) rrrrrZrrrr"r)r attrgetter) rk partitionrrcandidate_symbolsrrt symplified_srs @rl!get_graph_partition_symbol_inputsz+Scheduler.get_graph_partition_symbol_inputss X= X % X  -  % ,7Ijl6H6H Iy I7  !  N+:K:K:M N ++<=(2 "A77++44Q7L JJ|00 1# &(*=*=f*EFGGrnc g}ttjj}j }dfd t t |t |D]2\}}t}|D]+} |j| jj-|j|} tjj|D cgc]} | jc} } t| j| j zD cgc]} t#| t$s | j& c} |z } tfd| D} t}|D]} |j| j(||z Dcgc]}||vr| }}| j|| Dcic] }||vr|||}}| Dcic] }||vr|||v }}| Dcgc] }||vr||vr|}}| j|tfd| D} | Dcgc]}|s||}}| Dcgc]!}|tjj*vs |#}}j-||}t/||||||}|j1|| j3|| z }5|dddScc} wcc} wcc}wcc}wcc}wcc}wcc}wcc}w)z Gets signature for each graph partition, including input nodes, output nodes, and whether deallocating an input within graph partition. cjj|d}|yt|jjt r'j j|dx}r|Syy)z Checks if buf_name resolves to a NoneLayout buffer (following mutation_real_name). Buffers with NoneLayout are not allocated so graph partition should not take them as inputs or outputs. NFT)r@rrrr'r?r)rrris_unallocated_bufferrks rlrzFScheduler.get_graph_partition_signature..is_unallocated_buffersh ""&&x6C{#((//:6!% 7 7 ; ;Hd KK9K0;;rnc3VK|] }jj||"ywrwrrrrrks rlrz:Scheduler.get_graph_partition_signature..N,/1D''++D$71rc3VK|] }jj||"ywrwrrs rlrz:Scheduler.get_graph_partition_signature..xrrNr)rr rrv)rrZrrrrrrrlrrlr*rrrrrrr6rr`rprr<rr)rk partitionsskip_cudagraphsrunmet_output_namesrurr output_namesrreturned_output_namesrrJpartition_input_namesrrextra_input_namesrinput_deallocationextra_output_namesrr symbol_inputspartition_signaturers` @rlget_graph_partition_signaturez'Scheduler.get_graph_partition_signaturesc '(@(@(BC--/  ,*- Z (?";* %I~-7LL!##D$8$8$=$=$?@"%1$=$=>P$Q !'11<<.78id!!i8K"-!2!2[5G5G!G!GA)!W5!G "%//1/% ! 5?L !$++DOO<"2L@!@D<'@ ! " ( (): ;21D<'l4((1 2"1D<'d2221 "2"1D<'D8L,L1 " " ( (); <$./1/% !21D,T2T"1 "7!6$!''BSBS:S!6 !BB;M#:" #    1 2!6! J)"J.9J3J8 J=%!KKc|jjDcic]$\}}|tjjvr||&}}}|j jDcic]$\}}|tjjvr||&}}}|j Dcgc].}|jtjjvr|0}}|jDcgc] }|tjjvr|"} }t|j||||j| Scc}}wcc}}wcc}wcc}w)z Updates the partition signature by removing buffers specified in V.graph.removed_buffers. See [Note: Removed Graph Partition Arguments] ) rrrZrr?rrmaybe_get_namerr<rr) rkrrr]rr rrrrs rl.clean_removed_buffer_from_partition_signaturesz8Scheduler.clean_removed_buffer_from_partition_signaturessV!* 5 5 ; ; = = f177222 &L =  '99??A A c177222 #IA  ".. .""$AGG,C,CC .  "00 0177222 0  '  # #     $ $    )    s)D/')D5!3D;$%Ecp ddl t gg t|Dcic]\}}|| c}} d fd d fd }|D]5}t|jj  |< |dk(s. |7g}d}|t|kr sr} r0 j \}}|j||| r0r0 j \}}|j|||r0|dz }|t|kr rzr}|t|kDr td|Scc}}w)a Reorder nodes to minimize the number of partitions via a bfs topological sort. This is the optimal reordering such that the number of partitions cannot be reduced further. This may be sub-optimal for other metrics such as peak memory. This does not change relative orders of two cudagraphable nodes, nor the relative order of two non_cudagraphable nodes. rNc||f}j|rj|yj|yrw)rheappush)rnode_with_indexcudagraphable_nodesheapq node_to_indexnon_cudagraphable_nodesrks rlinsert_pending_nodeszHScheduler.reorder_for_minimizing_partition..insert_pending_nodess>,T2D9O$$T*6H2ODrnc|jjD]*}|dkDsJ|xxdzcc<|dk(s#|,yr=)re succ_nodes)r succ_nodernode_to_indegrees rlupdate_indegreezCScheduler.reorder_for_minimizing_partition..update_indegreesO!]]55 ' 2Q666 +q0+#I.!3(3 6rnrz Failed to schedule, while loop ran too long when reordering for minimizing the num of partitions rr_rrU) rrrrre pred_nodesheappoprrU)rkrrrrr num_itersrrrrrrrs` @@@@@@rl reorder_for_minimizing_partitionz*Scheduler.reorder_for_minimizing_partitionsV 9=CEGI4=e4DE4DysDs4DE  E E 4D%()A)A%B T "%*$T* -/ #e*$ #':)--(?@4%%* &--(;<4%%& NI#e*$ #': s5z ! ]Fs D2c Xddlm}m}ttj j }|||j|jttj jj|\}}|j|}||||\}} ||dzkr|S|S)zx Reorder nodes to minimize the number of partitions if this only slightly increase peak memory. r)estimate_peak_memoryprepare_planning_infor) rrrrrZrrr@rrrr) rkrrrrdefault_peak_memoryr5reordered_nodesreorder_peak_memoryrs rlrz0Scheduler.maybe_reorder_for_minimizing_partitions H"177#;#;#=> :O      # # qww++002 3  ; 77??F!5 7" Q !4s!: :" " rnc g}g}g}dd}|D]n}|j|du}|r*t|jdk(r|j|B|r||r|j|^|j|p||z|zS)a Reorder a node if it should be partitioned and has simple dependency: 1. move a partitioned node to the front if it has no dependency 2. move a partitioned node to the back if it is only used by OutputNode 3. otherwise do not reorder c|jD]0}|jD]}t|jtry2yrF)rzrrrrG)rrrHs rlonly_output_userzPScheduler.reorder_for_partition_with_simple_dependency..only_output_user.s9'')99C%chh ;$%*rnNrr )rrrqr)rkrfrontmiddlebackrrrs rlrz6Scheduler.reorder_for_partition_with_simple_dependency s*,*,(* D#44T:$F C(?(?$@A$E T"!&6t&< D! d#v~$$rncg}d}g}g}|jD]S}|j|du}|r)||k7r$|j||j|g}|}|j|U|r"|j||j|tjj }|dkDrXt t||D]@\}\} } | r td| D} | |ks$d||<tjd|| |B|j||} |j| |j|| || fS)z Given a list of BaseSchedulerNodes, split into a list of graph partitions and compute partition input/output signatures. TNrc3@K|]}t|tsdywrN)rrArs rlrz,Scheduler.graph_partition..`s#'!*A)!-CD!*rzFPartition %d has %d kernels, below minimum size %d, skipping cudagraph)rr)rrrr(rcudagraph_min_partition_sizerrrcudagraphs_logrrr_log_graph_partitions) rkrr cur_partitionrrnode_should_partitionmin_sizerrskip kernel_countrs rlrzScheduler.graph_partition@sd+- ') JJD$($9$9$$?t$K !3H!H!!-0&&~6 " 2N   &    m ,  " "> 2==== a<(1#j/2R(S$$It#&'!*'$L $h.-1*&,,d($ )T"77!?8  ))*5 "":z::%%rnc `tjtjsyt dt j jD}|sytd|D}t||z }tjdt|||tt||D]\}\}}tjd|t||jrdndt|jt|j|jsm|D]} |j!| y)Nc32K|]}t|ywrw)rU)rrs rlrz2Scheduler._log_graph_partitions..sO:NVF^:Nr3c3:K|]}|jrdywr)rrs rlrz2Scheduler._log_graph_partitions..s!PZq?O?O!ZszCCreated %d graph partitions: %d cudagraphable, %d non-cudagraphablez3 Partition %d: %d nodes, %s, inputs=%d, outputs=%dznon-cudagraphable cudagraphable)r rrrrrZr device_typesrrrrrrrr_log_non_cudagraphable_node) rkrrhas_gpu_devicecudagraphable_countnon_cudagraphable_countrrrrs rlr zScheduler._log_graph_partitionsws **7==9 O!'':N:NOO !!PZ!PP"%j/4G"G Q  O  #  *33z:3N)O %A% 9  EI'0'?'?#_I))*I**+  ''%D44T:&*Prnc|j|}|sy|j}|j|jjnd}d|g}t |jj }|j d||F|jddjd|jDd}|j d|tjd |dj||Z|jjd d}|r;|jjd D]} tjd | yyy) z)Log details for a non-cudagraphable node.Nzreason=zir=rrc32K|]}t|ywrw)r )rrs rlrz8Scheduler._log_non_cudagraphable_node..s2P?$7%. &  MM) $agg224PQ QQKKIVI8AAGG 5 GG - - /J ! 4 4 = =agg>R>R S NA/02 88^T 334F R &&--)2)?)? @)?T]]_)? @ 90 /: AsDH?H*H'cLtjdfd }|S)Nc3KjjrytjjrZjjJdt j jjjj djrGtjjr(t j jjd_y#jrGtjjr(t j jjd_wxYww)Ndevice should have an index) %update_graph_partition_default_devicervrKrrrZrr:codegen_device_guard_entercodegen_device_guard_exit)rrkrsrlctxz1Scheduler.use_default_device_context..ctxs  6 6z: N**/@++0002288D1D$$??//55 3..3D//444GG((BBD.2+ ..3D//444GG((BBD.2+s BEC; AE;AEE)rzIterator[None]) contextlibcontextmanager)rkrrr5 s``` rluse_default_device_contextz$Scheduler.use_default_device_contexts&  " " 3 # 3*u rnc t|dk(r|djsydd} dd}d}t||D]\}}|jr||}n|yt||D]\}}|js|||ry||_y)Nrrc4|dj}|J|Srr)rpartition_devices rlget_cudagraph_partition_devicezWScheduler.update_graph_partition_default_device..get_cudagraph_partition_devices'(|668 #/ //# #rnc@|D]}|j}||k7syyrFr)r target_devicerrs rlall_on_target_devicezMScheduler.update_graph_partition_default_device..all_on_target_devices*"*]* "rn)rr`rr6)rr`r> r6rrv)rrrrv)rkrrr< r? cudagraph_partition_devicerrs rlr2 z/Scheduler.update_graph_partition_default_devices z?a  1 (D(D  $  $ 5A   &*"$' J$? Iy++-KI-V*%@ & - $' J$? Iy''0D51 %@ 'A#rnc|j\}}t|dkDrtddxxt|z cc<|j||5t ||D]V\}}t|dk\sJdt||j r|j |E|j||X dddt|j}tjjj||dkDrqtjjJ|ttjjk(s.Jd|dttjjyy#1swYxYw) z Split nodes into partitions and codegen each partition into separate functions. This allows further applying different optimizations (e.g., cudagraph) to each function. rrcudagraph_partitionsz5Each partition must have at least one node but found NrzExpect z partition maps but got )rrrr8 rrr r. rrorZrr:set_all_partition_namesr)rkrrrrnum_partitionss rlr zScheduler._codegen_partitions0sY "&!5!5!7 J z?Q  Z !7 8C O K 8  , ,Z D(+J (C$ 99~*KCPYNK[\*++MM),33IyI)DEd;;< 44^D A 77))5 55!S)?)?%@@ .))A#aggF\F\B]A^_ @ E Ds A&E55E>cntjrddl}tj}t }t |D]}|jdk(r/|j|jjjk(rnQ|j|jf}||vs"Jd|jd|jd|j||j|_|j J|jrBtj"j$r(t&j(j*j-t&j(j*j/|D]}t0j3t4j6r4 t0j9d|j;|j=|jA|tjBrDt&j(j*jEd|jFjHD|jKx}rs||jk7s |jMs|jOr|jQ||jk7r$|jrctS|jjTrD|jV|jYt&j(j*j[||_tS|jTr|j\Jd d } |j_r5t |j`jc} | rte| d znd } t&j(j*jg|j\| |jh|j_r|j|jk|t&j(j*jmd |jFjHD||_7|jpjs|jt|jOrP|jwty|j{\} } } |j}|j| | | n5|jMr|j|n|jrxtjt|}|j}|}d d lEmF}d d lGmH}d dlImJ}t||||fr|}ntdtU||j|nt|tr!|j}|j|nYt|ttfr!|j}|j|n"t|tsJ|jtj"jr|j}|j|jjs|j|jjs|jt|tsP|jK}|>|jTdk7r/|j}|jr|jQtd|j{Dr ||_d|_|j|jk7rU|jJtS|jjTr(t&j(j*j[d|_|jQy#t>$r(t0j9d|j;YwxYw)Nr_compile_innerzDuplicate stack frame :zs; did you add a decorator to one of the functions in this stack trace? If so, try using a context manager instead.z5Generating code for node %s with estimated runtime %fz6Generating code for node %s with estimated runtime 0.0c34K|]}|jywrwrrs rlrz%Scheduler._codegen..sD(>CHH(>rr1 rc34K|]}|jywrwrrs rlrz%Scheduler._codegen..sC$:S$:r)CUDACombinedSchedulingrr)XPUCombinedSchedulingz type(self)=rYc3<K|]}t|tywrw)rrrs rlrz%Scheduler._codegen..sJ9IA:a/9Ir)]r("check_stack_no_cycles_TESTING_ONLYtorch._dynamo.convert_frame traceback extract_stackrrrfilename_dynamo convert_frame__file__linenorrvrrsrautotune_at_compile_timerZrr:write_get_raw_stream_headerregister_alignment_check_inputsrrrrrrrrr size_assertscodegen_deferred_input_assertsrrrr"r rurKrcurrent_stream_idxgenerate_stream_ctx_exitr4 rrrrrr3 rgenerate_stream_ctx_switching!codegen_deferred_alignment_copiesrtrrr`rrrrccodegen_templater{r$rGrHro codegen.cuda_combined_schedulingrJ rvrs#codegen.xpu.xpu_combined_schedulingrK rrcodegen_combo_kernelr8codegen_mix_order_reductionrr codegen_noderArdebug_sync_kernel codegen_syncrrr>rready_to_flushr)rkrrstackrIframer%rr num_streamsunique_streamsrrrbackend_rJ rsrK rQs rlr zScheduler._codegenPs  4 4 .++-E7A|D!%JJ"22%--*E*E*N*NN~~u||4$,U^^, <<>D . IIO 224   t $""$$CCD(,(8(8(>(>D**v*d111~~''')JJLT000**/@++000 22> 99;,,FFH*0D'(5%||7V9VV7&' 779-78K8K8R8R8T-UN;IN 3a 7q(,,GG"LL' ;;++-$2E2E2Q2248 GG B BC$($4$4$:$:C !%D   % % , ,T__ =!484W4W)*51-  (99!8X!((."{{#=tD++F3T8V#%;=RS'G(KDJ=)9::,,T2D"9:  (DDTJD#5}"EF  (55d;!$(>??? }}..  (557  ' ' . .t/D/D/F G  % % , ,T-E-E-G Hd$:;*& v-((0??AJJLJ9IJJ%)"%)"EH   $"="= =&&2 22 !4!4!9!9:$$>>@! O!IIP s3^-^43^4c|dj}|tj_||_|J|j |}|j ||S)rsr)rrZrrrrcbenchmark_combo_kernel)rkrnode_benchmark_resultsrrQs rlrn z Scheduler.benchmark_combo_kernelsZ1((* $!!!""6*--i9OPPrnc|}|djtfd|DsJdtjsyddlm}dg}}i}t |D]\}}|j} |j| rtjd |j| \} } | | f||<tj| rtjd|y || z }|j|  |j||\} }}| |z d kxs|d k}tj!t"j$rP|| kDs|r%tjdt'|| z dn$tjdt)|| z d| |z |kxs|S#|$r.} d t| vrtjd Yd } ~ yd } ~ wwxYw#|$r-} d t| vrtjd Yd } ~ yd } ~ wwxYw)rrc3DK|]}|jk(ywrwr)rrrs rlrz4Scheduler.speedup_by_combo_kernel..sK?44??$.?s z$$U! " 2IC   d #93< *.*E*E!7+ 'CKY,9c  " "7== 1SyL  E#)C2   Ic #0 Y$4 4Q$ *c!f4$$]     &#a&0  Y s=AF9GG""G  G  GH"H?HHcp|j|}|jJ|jjSrw)r@r get_layout)rkrrs rlget_buffer_layoutzScheduler.get_buffer_layout_s5x(xx###xx""$$rnc|jD]}|js|jjD]}tj j j|j}|s9t|dk(sHt|jttfri|jgk(s}tj jj!|jyr)rrUrrrZrrrrr;rr'r?r>rzero_dim_cpu_tensor_listr)rkrrHr]s rlruz$Scheduler.update_zero_dim_cpu_tensordsJJD{{} ,,22DWW3377 BF+F3u< *"MMJ8I+J!#OO-388<":. .rnct|trJ|j|}tjj j ||_y)z6Code-gen to enter the Stream context assigned to node.)rN)rrArrZrr:codegen_cuda_stream_enterr)rkr node_streams rlgenerate_stream_ctx_enterz#Scheduler.generate_stream_ctx_entersId$:;;;))$/ #$77#7#7#Q#Q"$R$  rnc~|jJtjjj d|_y)z1Code-gen to exit from the current Stream context.N)rrZrr:codegen_cuda_stream_exitrjs rlr\ z"Scheduler.generate_stream_ctx_exits2''333 557#' rnc&||jvsJt|trdn|j|}|j|k(ry|j|y|j||j |y|j |j |y)amGenerate stream entering and exiting to properly run node in a multi-stream scenario. Stream context switching is only generated if ``node``'s assigned stream is different from the previous node's stream. NopKernelSchedulerNodes have stream=None and inherit the enclosing stream context (or do nothing if no context is active yet). N)rrrAr[ r r\ )rkrrFs rlr] z'Scheduler.generate_stream_ctx_switchingst*****$ 67 $$T*   " "f ,   $ $ 0V^   $ $ ,1C  * *4 0  ) ) +  * *4 0rn)rzlist[ir.Operation]rrU)rz!dict[str, SchedulerDonatedBuffer]rTrV)rr rr )rr rr_rrvrX)rrYrrU)rr rrU)rrrr_)r str | Nonerrvr)rr_rr)rrrrrtuple[float, str]rwrrr{rvrzrrr )r~rrr6rr )rir.MultiTemplateBufferrrv) rir.OperationBufferrr rr rrrrU)rrrrv)rrrzrrz&tuple[LambdaFuture | None, ModuleType])rr_rr_rrd)rr_rr_)rr_rr_rOrderedSet[BaseSchedulerNode]rr_)rr_rr_rrxrr )r,dict[BaseSchedulerNode, list[PendingFusion]]rr rrU) r,1list[tuple[BaseSchedulerNode, BaseSchedulerNode]]r*&dict[BaseSchedulerNode, PendingFusion]r-r rr rlrv)rr r*r )r6r r7r )rrrlrvrr)rXrrrUrN)rrrlrvrr r )rr_rr_rr rrv)rr_rr_rz!tuple[str, ...] | OrderedSet[str]rr rr )rr_rr_rrrrv)rr_rr_rz,tuple[int, SchedulerNode, sympy.Expr] | None)FT) rr_rr_rrvr<rvrrv)r r6rr_rr_rrv)rHr3rr4rrv)rHr3rr5r*rhrrvr)rr3rErvrr )TFT) rr_rr_rErvr1rvr<rvrzint | tuple[int, int, bool])r6r rr )rrrr)rzr_rrU)rr6rBaseScheduling)rrYrr r)rr rr_rrv)rr_rr )rr)r6dict[str, ir.IRNode | ir.TorchBindObject | sympy.Expr])rlist[GraphPartitionSignature]rrU)rr`rr rr)rlist[PartitionType]rz list[bool]rr )rr<rr<)rz9tuple[list[PartitionType], list[GraphPartitionSignature]])rr rr rrU)rr`rr<rrU)rr rr rz'contextlib.AbstractContextManager[None]rrrz%tuple[float, float, list[str | None]])rrrrv)rr rz ir.Layoutr)rr )oryrzr{rrxrNrwrrrrBpropertyrsetterrrrrrr]rr|rDr{rrXrr}r~rrrrtrxr}rrrrrrr2rrr$r0r4r<rmrrr?rrurrrrrrOrrrrr5rrrrIr:r;rQrRrrsrur{rrcrr~rrIrrrrrrrrrrrr r r;r. r8 r2 r r rn rCry rur[ r~ r r\ r] rSrTs@rlrr s6 o9b #6 p(S Q&&((7# ,"HMP^KZ+#Z , 6 S*4#&/<$6!F 808 8,%) *"  & >  >*6 >  >)0) )VVDp'8&'8+'8  '8  '8  '8R OS0AK / P&P/@P Pd > !3   0 !'  3 "D2$PD23D2  D2LO?PO?@O? L O? 3 O?  O?bS2S@S0$K$$U$(C&CC ! CJ3.j?6 &6 6  ; 6 p,&,/@, ,\7&7/@7 7r.2&.2/@.2MP.2 .2`$&$/@$ $6< <!<< < <|M&M/@M M^[ &[ /@[ [ z ;(;); ;  ;z`&`/@` 5`L"*. zM zM!zM zM $( zM  zMx;&;/@; ;z3)3)(93)BS3) 3)r%T   ' 7J   ;!.3*. mK mK!mK mK (, mK $( mK %mK^d d!d  dLR &R /@R R h6 Q6 :6@4@4 4 8*4 ) &'*% 5$    +:    =~!!F% " ? '1' 'RBH BHLBH " BHHK -K @JK &K Z" 0" " H?&? !?B& !>%,% %@5& B5&n";'";2";  ";H>0)  ) +)   ) V-;X 06-A--A;X-A -A^@ob Q4 Q . QN5`% H ( 1rnc<eZdZdfd ZddZddZ ddZ ddZ ddZ ddZ ddZ dd Z d dd Z dd Z dd Zdd Zd dZddZ d!dZd"dZ d#dZ d$dZ d d%dZxZS)&r c0t|||_yrw)r>rxr)rkrrGs rlrxzBaseScheduling.__init__s "rncR|jr|jjyyrw)rrsrjs rlfree_buffers_in_schedulerz(BaseScheduling.free_buffers_in_schedulers >> NN ' ' ) rnctS)z0Return a set of .codegen.common.BackendFeature()rrs rlget_backend_featuresz#BaseScheduling.get_backend_featuress |rnct)zO Check whether node1 and node2 can be vertically fused or not. r*rs rlrz BaseScheduling.can_fuse_vertical "!rnct)zQ Check whether node1 and node2 can be horizontally fused or not. r*rs rlrz"BaseScheduling.can_fuse_horizontalr rnc|j}t|tjsy|j syt|j tj rt|j jdk(xrkt|j jdtjxr8|j jdj|jk(Sy)av A Multi-Output Template (referenced in #144012) is a template node with MultiOutputLayout, and its output buffers are instances of MultiOutput. In this context, we verify whether node1 represents the Multi-Output Template and node2 corresponds to one of its outputs. If so, we further check if backend supports this fusion. Frr) rrr+rarVrr=rrrr)rkrrrs rlrz.BaseScheduling.can_fuse_multi_outputs_templates..0 ,(9(9:557 ejj".. 1EJJ%%&!+Ouzz003RYY?OJJ%%a(113|7L7L7NN  rnc|js|jrtj||Stj ||r t ||St |t r|j|St |trLt |tr  UBL GG > >\  rn)rzScheduler | NonerT)rr6rzOrderedSet[BackendFeature]r r4)r~rQrz"tuple[tuple[sympy.Expr, ...], ...])rr_r rrrrr rwr )rz"FusedSchedulerNode | SchedulerNoderrU)rr8rrUrVr )r~rrr rr )r rrr rrU)ryrzr{rxr r rrrrrrdr_ rxrd rc rf rg rurtr}rgrn r rSrTs@rlr r s#*"&"/@" ""&"/@" "&/@ 49&9/@9 9("3" +" "( "4 "4 "  "$%) "* " "" " "" "" " "0" ""&/@ "4" ."#'2   rnr )rz$torch._inductor.codecache.LocalCache)rr_rr )rr_rzCallable[[Any], Any] | None)rr_rrm)r,rrr )rr_rrr@rkrrU)r)FusedSchedulerNode | GroupedSchedulerNoderrU)rr rrrrrrU)r~)rzlist[list[int]]r~rrrrz list[int])rr rr rrUr) r r r r r r r r rrCrztuple[int, int])rrrrr r r r r r r r rrCrrvrV)rz ir.IRNoderr)rr_rr)rr_rrv)rr_rr_) __future__rrr6 rZrrrrrBrrr*r+rO rGrrconcurrent.futuresrrrr r r r typing_extensionsr torch.utils._ordered_setrr+rrcollections.abcrrrtypesrtorch._inductor.codegen.wrapperrr7rrrtorch._inductor.async_compiletorch.utils._pytreer_pytreertorch._dynamo.utilsrr torch._inductor.autotune_processrtorch._inductor.codecacherrtorch._inductor.irrtorch._inductor.metricsrr torch._inductor.stream_utilsr!%torch.fx.experimental.symbolic_shapesr"torch.utils._sympy.symbolr#r$r%torch.utils._tritonr&rr'r(r)r*r,analyze_preserves_zero_maskr-codegen.commonr.r/r0 comm_analysisr1r2r3r4r5r6excr7r8fx_utilsr9r:r;r<r=r>r?rr@rrArB runtime.hintsrCrDruntime.runtime_utilsrErFrrGrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrY virtualizedrZ getLoggerryr_logginggetArtifactLoggerrrrr rr`r|rarb dataclassrdrrrr]r_rrrrrrr)rGrrrArrrrr8rVrorrrrrrr]rnrmr'r-r2r=rBrDrGrIrKrr r~rnrlr s"   ,3BB'/)<< J5 $$$6E?7M8>OO*DDDMM;:2$ J:7&(g! ^^ - -h A NN44XO >>;; $11(LI 34 y4 T]t_ DDD* (((YYx h8h8h8V 1_11H 1H 1V 22, ' #LT"  # *   *K *K4*K,*K *KZ"* 1"*J5.5S*%S*l : $: $ 8k**k*\[G0[G|:-+=:-zC:!3C:L b,bP#%+#+ + + +\0%01C0 08 1(((( ( # (  (6       # @    >+9??,4$ &""P@nJ1nJ1bUxxrn