+
    &jtv                    o   ^ RI t ^ RIHtHt ^ RIHt ^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIt^ RIHt ^ RIHtHtHt ^ RIHtHtHtHt ^ R	IHt ^ R
IHtHt ^ RIHtHtHtH t H!t!H"t"H#t#H$t$H%t%H&t&H't'H(t(H)t) ^ RI*H+t+H,t,H-t-H.t.H/t/ ^ RI0H1t1H2t2 ^ RI3H4t5 ^ RI6H7t7H8t8 ^ RI9H:t; ]	! R4      t<]! R4      t=]P|                  P~                  t?]P                  P                  RRR4      tB]C! ^4      w  tDtEtFR tGR tHR R ltIR R ltJR tKR tL]I! ]?P                  ]?P                  .4      ]/! 4       RRR]P                  RR3R l4       4       tP]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R  4       4       tT]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R!ER/R" l4       4       tU]I! ]?P                  4      ]/! 4       R# 4       4       tV]I! ]?P                  P                  ]?P                  P                  ]?P                  P                  ]?P                  P                  .4      ]/! R$R%4      R& 4       4       tY]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R' 4       4       tZR( t[ERR) R* llt\]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R+ 4       4       t^^t_R, R- lt`]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R. 4       4       tb]I! ]?P                  P                  4      R/R/R0 l4       te]I! ]?P                  P                  4      R1]P                  R2RR3RR4R/R5 l4       tg]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R1]P                  R2RR3RR4R/R6 l4       4       ti]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R1]P                  R2RR3RR4R/R7 l4       4       tl]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R1RR2RR3RR4R/R8 l4       4       tn]I! ]?P                  P                  4      R9 4       tp]I! ]?P                  P                  4      R: 4       trR; ts]I! ]?P                  P                  4      ERR< l4       tu]I! ]?P                  P                  4      ERR= l4       tw]I! ]?P                  P                  ]?P                  P                  .4      ]/! 4       R> R? l4       4       ty]I! ]?P                  P                  4      ERR@ l4       t{RA t|]I! ]?P                  P                  4      RB 4       t~]I! ]?P                  4      ERRC RD ll4       t]I! ]?EP                  4      ERRE RF ll4       t]I! ]?EP                  4      RG^RH^RIR/RJ RK ll4       t]I! ]?EP
                  4      ERRL RM ll4       t]I! ]?EP                  P                  4      RNRO/RP RQ ll4       t]I! ]?EP                  P                  4      RNRO/RR RS ll4       t]/! 4       ]I! ]?EP                  P                  4      RT 4       4       t]I! ]?EP                  P                  4      RURR%RRVRRW^ RXRRYR/RZ R[ ll4       t]I! ]?EP                  P                  ]?EP                  EP                   .4      ]/! 4       R\ 4       4       t]I! ]?EP                  EP$                  4      ERR] l4       t]I! ]?EP(                  P                  ]?EP(                  EP                   .4      ]/! 4       R^ 4       4       t]I! ]?EP(                  EP$                  4      ERR_ l4       t]I! ]?EP.                  P                  4      R` 4       t]I! ]?EP.                  P                  4      Ra 4       t]I! ]?EP4                  P                  4      Rb 4       t]I! ]?EP4                  EP8                  4      Rc 4       t]I! ]?EP<                  P                  4      Rd 4       t]I! ]?EP@                  P                  4      R1RR2RR3RR4RReR/Rf l4       t]I! ]?EPD                  P                  4      ER	Rg l4       t]I! ]?EPF                  P                  4      ERRh l4       t]I! ]?EPJ                  P                  4      ER	Ri l4       t]I! ]?EPL                  P                  4      Rj 4       t]I! ]?EPP                  EP8                  4      Rk 4       tRl Rm ltRn Ro ltER
Rp Rq lltERRr Rs lltRt Ru ltERRw Rx lltRy Rz lt]I! ]?EPb                  P                  ]?EPb                  EPd                  .4      ]/! R{R|4      ERR~ R ll4       4       t]I! ]?EPh                  P                  ]?EPj                  P                  .4      ]/! 4       R R l4       4       t]I! ]?EPn                  .4      ]/! R{R|4      R R l4       4       tR R lt]I! ]?EPt                  4      ]/! 4       R R l4       4       t]I! ]?EPv                  4      ]/! 4       ERR R ll4       4       t]I! ]?EPx                  4      ]/! 4       ERR R ll4       4       t]I! ]?EPz                  4      ]/! 4       ERR R ll4       4       t]I! ]?EP|                  P                  4      ERR R ll4       t]I! ]?EP~                  P                  ]?EP~                  P                  .4      ]/! 4       R R l4       4       t]I! ]?EP                  P                  4      ERR R ll4       t]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! RRR4      RRRR/R R ll4       4       t]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       RR/R R ll4       4       t]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! RR}R4      RRO/R R ll4       4       t]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! RRR4      RRORR/R R ll4       4       t]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       RRORR/R R ll4       4       t]I! ]?EP                  4      ]/! RR}R4      ERR R ll4       4       tR R lt]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! RR4      ERR R ll4       4       t]I! ]?EP                  P                  ]?EP                  EP                  .4      ]/! RRRR4      R R l4       4       t]I! ]?EP                  P                  4      ERR R ll4       tR R ltR R ltR R lt]I! ]?EP                  4      RRORRRvRRRRRRR/R R ll4       t]I! ]?EP                  P                  ]?EP                  P                  .4      RRORRRR/R R ll4       t]I! ]?EP                  4      ]/! RRROR7      ERR R ll4       4       t]I! ]?EP                  P                  4      R 4       t]I! ]?EP                  4      ]/! 4       ERR R ll4       4       tR tR t]I! ]?EP                  4      ]/! 4       R 4       4       t]I! ]?EP                  4      ]/! 4       R 4       4       tR t]I! ]?EP                  4      ]/! R4      R 4       4       t]I! ]?EP                  4      ]/! R4      R 4       4       tR t]I! ]?EP                  4      ]/! 4       R 4       4       t]I! ]?EP                  4      ]/! 4       R 4       4       t]I! ]?EP                  P                  4      R 4       t]I! ]?EP                  P                  ]?EP                  EP                  ]?EP                  P                  ]?EP                  EP                  .4      ]/! R4      R 4       4       tR t]I! ]?EP                  4      ]/! 4       R 4       4       t]I! ]?EP                  4      ]/! 4       R 4       4       t]I! ]?EP                  P                  ]?EP                  EP                  ]?EP                  P                  ]?EP                  EP                  .4      ]/! R4      R 4       4       t]I! ]?EP                  4      ]/! 4       ERR R ll4       4       t]I! ]?EP                   4      ]/! 4       R R l4       4       Et]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! ROR7      RH^RG^/R l4       4       Et]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       R/R/R l4       4       Et]I! ]?EP                  EP                  4      ERR l4       Et]I! ]?EP                  EP                  4      ERR l4       Et
]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       ERR l4       4       Et]I! ]?EP                  P                  4      ERR l4       Et]I! ]?EP                  4      ]/! ROR7      ERR R ll4       4       EtR EtR R lEtERR R llEtR Et]I! ]?EP*                  P                  4      R R l4       Et]I! ]?EP.                  P                  4      R R l4       Et]EP2                  EP4                  '       Ed   ]P                  P                  RRR4      Et]I! ]P|                  EP8                  EP:                  P                  4      R 4       Et]I! ]P|                  EP8                  EP>                  P                  4      R 4       Et ]EP2                  EPB                  '       dN   ]P                  P                  RRR4      Et"]I! ]P|                  EPF                  EPH                  4      R 4       Et%]P                  P                  RRR4      Et&]I! ]P|                  EPN                  EPP                  P                  4      ]I! ]P|                  EPN                  EPR                  P                  4      ]I! ]P|                  EPN                  EPR                  EPT                  4      R 4       4       4       Et+]I! ]P|                  EPN                  EPP                  EPX                  4      ]I! ]P|                  EPN                  EPP                  EPZ                  4      R 4       4       Et.]I! ]P|                  EPN                  EP^                  P                  4      ]I! ]P|                  EPN                  EP^                  EPT                  4      R 4       4       Et0]I! ]P|                  EPN                  EP^                  EPX                  4      ]I! ]P|                  EPN                  EP^                  EPZ                  4      R 4       4       Et1]I! ]P|                  EPN                  EPd                  P                  4      ]I! ]P|                  EPN                  EPf                  P                  4      R 4       4       Et4]P                  P                  RRR4      Et5]I! ]P|                  EPl                  EPn                  4      ERR l4       Et8]I! ]P|                  EPl                  EPr                  4      R 4       Et:ER  Et;]I! ]?EPx                  4      ER ER l4       Et=]I! ]?EP|                  P                  4      ERER l4       Et?ER Et@]I! ]?EP                  P                  4      ER 4       EtB]I! ]?EP                  4      ]/! 4       ERER l4       4       EtD]I! ]?EP                  4      ]/! R4      ER 4       4       EtF]I! ]?EP                  P                  4      ER 4       EtH]I! ]?EP                  P                  4      ER	 4       EtJ]I! ]?EP                  P                  4      ER
 4       EtL]I! ]?EP                  4      ]/! R4      ER 4       4       EtNER ER lEtO]I! ]?EP                  4      ]/! RR%4      ER 4       4       EtQ]I! ]?EP                  4      ]/! R4      ER 4       4       EtS]I! ]?EP                  4      ]/! RR%4      ER 4       4       EtU]I! ]?EP                  4      ]/! R4      ER 4       4       EtW]I! ]?EP                  P$                  4      ERER l4       EtY]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       ER 4       4       Et[]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       ERER/ER ER ll4       4       Et\]I! ]P|                  P~                  EP                  P                  ]P|                  P~                  EP                  P                  .4      ]/! 4       ER 4       4       Et]]I! ]?EP                  P$                  ]?EP                  P$                  .4      ER 4       Et`]I! ]?EP                  P                  .4      ER 4       Etb]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! ROR7      RH^RG^/ER l4       4       Etd]I! ]?EP                  P$                  .4      ER 4       Etf]I! ]?EP                  P                  ]?EP                  P                  .4      ERRERR/ER l4       Eti]I! ]?EP                  P                  .4      ERRERR/ER l4       Etk]I! ]?EP                  .4      ]/! 4       ER  4       4       Etm]I! ]?EP                  .4      ER! 4       Eto]I! ]?EP                  .4      ER" 4       Etq]I! ]?EP                  .4      ER# 4       Ets]I! ]?EP                  .4      ER$ 4       Etu]I! ]?EP                  .4      ER% 4       EtvER& ER' lEtwER( Etx]I! ]?EP                  .4      ER) ER* l4       Etz]I! ]?EP                  .4      ER+ 4       Et|]I! ]?EP                  .4      ER, 4       Et~]I! ]?EP                  P                  4      ER- 4       Et]I! ]?EP                  4      ]/! 4       ER. 4       4       Et]I! ]?EP                  P                  4      ERER/ l4       Et]I! ]?EP
                  P                  4      ER0 4       EtER
ER1 lEt]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       ERR1R/ER2 ll4       4       Et]I! ]?EP                  P                  ]?EP                  P                  .4      ER3 4       Et]I! ]?EP                  EP$                  ]?EP                  EP                  ]?EP                  EP$                  ]?EP                  EP                  ]?EP                  P                  ]?EP                  EP                  .4      ]/! R$R%4      ERER4 l4       4       Et]I! ]?EP"                  P                  4      ER5 4       Et]I! ]?EP&                  P                  4      ER6 4       Et]I! ]?EP*                  P                  4      ER7 4       Et]I! ]?EP.                  EP0                  ]?EP2                  EP0                  ]?EP.                  P$                  ]?EP2                  P$                  ]?EP4                  P                  ]?EP6                  P                  ]?EP8                  P                  .4      ER8 4       Et]I! ]?EP<                  EP0                  ]?EP>                  EP0                  ]?EP<                  P$                  ]?EP>                  P$                  .4      ERER9 l4       Et]I! ]?EPB                  EP0                  ]?EPD                  EP0                  .4      ERER: l4       Et]I! ]?EPH                  P                  ]?EPH                  EPJ                  .4      ER; 4       EtER< Et]I! ]?EPP                  P$                  ]?EPP                  EP0                  .4      ER= 4       Et]I! ]?EPT                  P$                  ]?EPT                  EP0                  .4      ER> 4       Et]I! ]?EPX                  P                  4      ER? 4       Et]I! ]?EP\                  P$                  ]?EP\                  EP0                  .4      ER@ 4       Et]I! ]?EP`                  P$                  ]?EP`                  EP0                  .4      ERA 4       Et]I! ]?EPd                  P                  4      ERB 4       Et]I! ]?EPh                  P$                  4      ]/! 4       ERERC ERD ll4       4       Et]I! ]?EPl                  .4      ]/! 4       ERERE l4       4       Et]I! ]?EPp                  .4      ERERF l4       Et]I! ]?EPt                  .4      ERERG l4       Et]I! ]?EPx                  P                  ]?EPz                  P                  .4      ERERH l4       Et]I! ]?EP~                  EP0                  4      ERI 4       Et]I! ]?EP                  P                  4      ERJ 4       Et]I! ]?EP                  4      ERK 4       Et]I! ]?EP                  4      ]/! 4       ERL 4       4       Et]I! ]?EP                  4      ERM 4       Et]I! ]?EP                  P                  4      ERERN l4       EtER	ERO lEt]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! ROR7      ERP 4       4       Et]I! ]?EP                  EP                  ]?EP                  EP                  .4      ]/! ROR7      ERQ 4       4       EtERR EtERS EtERT EtERU EtERERV ERW llEtERX EtERY ERZ lEtER[ Et]I! ]?EP                  4      ]/! R4      ER\ 4       4       Et]I! ]?EP                  P                  4      ERER] l4       Et]I! ]?EP                  P                  4      ER^ 4       Et]I! ]?EP                  4      ]/! RR%4      ERER_ l4       4       Et]I! ]?EP                  4      ]/! R4      ER` 4       4       EtERa ERb lEt ! ERc ERd]4      EtERe ERf lEt]I! ]?EP                  P                  4      ERg 4       Et]I! ]?EP                  4      ]/! 4       ERh 4       4       Et]I! ]?EP                  4      ]/! RERi4      ERj 4       4       Et]I! ]?EP                  P                  .4      ERk 4       Et]I! ]?EP                  P                  4      ERERl l4       Et]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       R1RR2RR3RR4RERmR/ERn l4       4       Et]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       R1RR2RR3RR4RERmR/ERo l4       4       Et]I! ]?EP                  P                  4      ERp 4       Et]I! ]?EP                  P                  4      ERERq l4       EtERr EtER
ERs ERt llEtERu EtERv Et]I! ]?EP                  P                  4      ERERw l4       EtERERx lEtERERy lEtERz EtERER{ lEtER ER| lEt]I! ]?EP                  P                  4      ER} 4       Et ]I! ]?EP                  4      ER~ 4       Et]I! ]?EP                  EP                  ]?EP                  EP
                  ]?EP                  EP                  ]?EP                  EP                  .4      ]/! 4       ERER l4       4       Et]I! ]?EP                  EP                  ]?EP                  EP
                  ]?EP                  EP                  ]?EP                  EP                  .4      ERER l4       Et
]I! ]?EP                  P                  .4      ER!ER ER ll4       Et]I! ]?EP                  EPl                  .4      ER!ER ER ll4       EtER ER lEt]I! ]?EP                  .4      ER!ER ER ll4       Et]I! ]?EP"                  .4      ER"ER ER ll4       Et]I! ]?EP&                  4      ERR/ER ER ll4       Et]I! ]?EP*                  .4      ERER ER ll4       Et]I! ]?EP.                  .4      ER#ER ER ll4       Et]I! ]?EP2                  .4      ER	ER ER ll4       Et]I! ]?EP6                  .4      ER$ER ER ll4       Et]I! ]?EP:                  .4      ER%ER ER ll4       Et]I! ]?EP>                  .4      ER&ER ER ll4       Et ]I! ]?EPB                  .4      ERER ER ll4       Et"]I! ]?EPF                  P                  .4      ER'ER ER ll4       Et$]I! ]?EPJ                  P                  .4      ER'ER ER ll4       Et&]I! ]?EPF                  EPl                  .4      ERER ER ll4       Et']I! ]?EPP                  .4      ERER ER ll4       Et)]I! ]?EPT                  .4      ER(ER ER ll4       Et+]I! ]?EPX                  .4      ER ER ER ll4       Et-ER)ER ER llEt.]I! ]?EP^                  P                  .4      ER)ER ER ll4       Et0ER*ER ER llEt1]I! ]?EPd                  P                  .4      ER)ER ER ll4       Et3]I! ]?EPh                  EPj                  ]?EPh                  EPl                  .4      ]/! 4       ER
ER l4       4       Et7]I! ]?EPp                  EPj                  4      ER
ER l4       Et9]I! ]?EPt                  P                  ]?EPt                  P                  .4      ]/! 4       ERR/R/ER ll4       4       Et;ER Et<ER Et=]I! ]?EP|                  P                  ]?EP~                  P                  .4      ERER l4       Et>]I! ]?EP                  P                  ]?EP                  P                  .4      ER	ER l4       Et@]I! ]?EP                  P                  ]?EP                  P                  .4      ER	ER ER ll4       EtB]I! ]?EP                  P                  ]?EP                  P                  .4      ERER l4       EtD]I! ]?EP                  P                  ]?EP                  EP                  ]?EP                  EP                  ]?EP                  EP                  .4      ER+ER l4       EtIER EtJ]I! ]?EP                  P                  4      ER	ER l4       EtL]I! ]?EP                  P                  4      ER 4       EtM]I! ]?EP                  P                  4      ER 4       EtN]I! ]?EP                  P                  4      ER 4       EtOER EtPER EtQ]I! ]?EP                  P                  ]?EP                  P                  .4      ERER l4       EtT]I! ]?EP                  P                  4      ER,ER l4       EtU]I! ]?EP                  P                  4      ER-ER l4       EtW]I! ]?EP                  4      ]/! 4       ER.ER l4       4       EtY]I! ]?EP                  P                  ]?EP                  EP                  .4      ]/! R$R%4      ERER l4       4       Et[]EP                  Et]ER Et^]I! ]?EP                  P                  4      ER 4       Et_]I! ]?EP                  P                  4      ER 4       Et`]I! ]?EP                  P                  4      ER 4       Etb]I! ]?EP                  P                  4      ER 4       Etc]I! ]?EP                  P$                  ]?EP                  EP                  .4      ]/! 4       ERRERR/ER l4       4       Etf]I! ]?EP                  EP0                  ]?EP                  EP                  .4      ERRERR/ER ER ll4       Eth]I! ]?EP                  .4      ]/! 4       ER/ER l4       4       Etj]I! ]?EP                  P                  ]?EP                  P                  ]?EP                  P                  .4      ER	ER l4       Etn]I! ]?EP                  P                  ]?EP                  P                  .4      ER	ER l4       Etq]I! ]?EP                  P                  4      ER 4       Etr]I! ]?EP                  P                  ]?EP                  P                  .4      ]/! 4       ERER l4       4       Ets]I! ]P|                  P~                  EP                  4      ER 4       Ett]I! ]P|                  P~                  EP                  4      ER 4       Etu]I! ]?EP                  4      ]/! 4       ERRERRERRERR/ER l4       4       EtwER Etx]I! ]?EP                  4      ER 4       Etz]I! ]?EP                  4      ER0ER l4       Et|]I! ]?EP                  4      ER0ER l4       Et~]I! ]?EP                  4      ER0ER l4       Et]I! ]?EP                  4      ]/! 4       ERRERR/ER l4       4       Et]I! ]?EP                  4      ]/! 4       ER ER l4       4       Et]I! ]?EP
                  4      ER ER l4       Et]I! ]?EP                  4      ]/! ROR7      ER ER l4       4       Et]I! ]?EP                  4      ]/! 4       ER ER l4       4       EtER EtER*ER ER llEt]I! ]?EP                  4      ]/! 4       ERER ER ll4       4       Et]I! ]?EP                  .4      ER*ER ER ll4       Et]I! ]?EP                  EP0                  4      ER1ER l4       Et]I! ]?EP"                  4      ]/! 4       ER ER l4       4       Et]I! ]?EP&                  4      ]/! 4       ERER l4       4       Et]I! ]?EP*                  4      ]/! 4       ER2ER ER ll4       4       Et]I! ]?EP,                  P                  4      ER3ER ER ll4       EtER Et]I! ]?EP2                  P                  4      ER4ER l4       EtER EtE]! ]?EP8                  4       E]! ]?EP:                  4       E]! ]?EP<                  4       E]! ]?EP>                  4       E]! ]?EP@                  4       E]! ]?EPB                  4       E]! ]?EPD                  4       E]! ]?EPF                  4       E]! ]?EPH                  4       E]! ]?EPJ                  4       E]! ]?EPL                  4       E]! ]?EPN                  4       E]! ]?EPP                  4       E]! ]?EPR                  4       E]! ]?EPT                  4       E]! ]?EPV                  4       E]! ]?EPX                  4       E]! ]?EPZ                  4       E]! ]?EP\                  4       E]! ]?EP^                  4       E]! ]?EP`                  4       ER Et]I! ]?EPd                  4      ]/! 4       ER 4       4       Et]I! ]?EPf                  4      ]/! 4       ER^/ER  l4       4       Et]I! ]?EPh                  4      ]/! 4       ER^/ER l4       4       EtE]! ]?EPd                  4      EtE]! ]?EPf                  4      EtE]! ]?EPh                  4      Et^ RI0t^ REIt^ REItER EtE]! 4        R# (5      N)CallableSequence)Enum)wraps)TypeVar)	ParamSpec)SymBoolSymFloatTensor)_add_op_to_registry_convert_out_paramsglobal_decomposition_table
meta_table)
OpOverload)_prim_elementwise_meta$ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND)BoolLikecorresponding_complex_dtypecorresponding_real_dtypeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KIND	FloatLikeIntLikemake_contiguous_strides_forNumber
NumberTypesuggest_memory_formatsym_min
TensorLike)_maybe_convert_to_dtype_maybe_resize_out_resize_output_check_safe_copy_outout_wrapper)_broadcast_shapes_maybe_broadcast)_config)ScalingTypeSwizzleType)_pytree_T_PatenIMPLMetac                 .    W,           ^,
          V,          #     abs   &&q/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/torch/_meta_registrations.pyceil_divr8   9   s    EAI!    c                <    W,           ^,
          V,          V,          # )z$Rounds up x to nearest multiple of yr3   xys   &&r7   round_upr>   =   s    UQY1!!r9   c                    V ^8  d   QhR\         \         \        \        3,          .\         \        \        3,          3,          /#    return)r   r,   r+   )formats   "r7   __annotate__rD   B   s2     
 
8BF#3"4hr2v6F"FG 
r9   c                    a  V 3R  lpV# )c                 Z   <a  \        S 4      o V 3R  lp\        P                  ! VS4       S # )c                 *   < \        \        V S4       R # N)r   r   )opfns   &r7   register0register_meta.<locals>.wrapper.<locals>.registerF   s    
B3r9   )r   pytree	tree_map_)rJ   rK   rI   s   f r7   wrapperregister_meta.<locals>.wrapperC   s)     $	4 	2&	r9   r3   )rI   rO   s   f r7   register_metarQ   B   s     Nr9   c                $    V ^8  d   QhR\         /# )rA   type_promotion)r   )rC   s   "r7   rD   rD   O   s      3r9   c                     \         P                  ! VR V / w  r#V Uu. uF  p\        WC4      NK  	  pp\        V!  p\	        VR\
        P                  / # u upi )type_promotion_kindrS   )utilsr   r    r&   r   r   DEFAULT)rS   args_result_dtyper<   s   $*   r7   elementwise_metar[   O   sp    
 ..	*OA ?CCd#A4dDC T"D "	BJJ  Ds   Ac                     \         P                  \         P                  \         P                  \         P                  \         P
                  \         P                  /pVP                  W 4      # rH   )torch	complex32halfcfloatfloatcdoubledoubleget)dtypefrom_complexs   & r7   toRealValueTyperg   c   sC    ekku||L
 E))r9   c                 t   a a \        \        S .VO5!  4      o\        P                  ! SS 8H  VV 3R  l4       R# )c                     < R S RS  2# )zoutput with shape z# doesn't match the broadcast shape r3   )broadcasted_shape
self_shapes   r7   <lambda>)check_inplace_broadcast.<locals>.<lambda>p   s    $ZL0STeSfgr9   N)tupler%   r]   _check)rk   
args_shaperj   s   f*@r7   check_inplace_broadcastrq   l   s0    /
HZHI	LLZ'gr9   Fc	           	      >  a aaaa	 \        S \        P                  4      '       d*   \        P                  ! S P	                  4       ^ 8H  R 4       \        S\        P                  4      '       d*   \        P                  ! SP	                  4       ^ 8H  R 4       \
        ;QJ d    R S SS3 4       F  '       g   K   RM	  RM! R S SS3 4       4      '       db   \        P                  ! \        P                  ! 4       4      o	Sf   S	oMP\        P                  ! \        P                  ! S4      V	V3R l4       MS;'       g    \        P                  ! 4       o\        S\        P                  4      '       g   \        R\        S4       24      h\        P                  ! \        S\        4      VV V3R l4       \        S\        4      '       g   \        R	\        S4       24      h\        P                  ! S^ 8  R
 4       \        P                  ! S3SVRVVR7      # )r   c                      R # z:linspace only supports 0-dimensional start and end tensorsr3   r3   r9   r7   rl   (meta_linspace_logspace.<locals>.<lambda>       Pr9   c                      R # rt   r3   r3   r9   r7   rl   ru      rv   r9   c              3   B   "   T F  p\        V\        4      x  K  	  R # 5irH   )
isinstancecomplex).0args   & r7   	<genexpr>)meta_linspace_logspace.<locals>.<genexpr>   s     
C/B:c7##/Bs   TFc                     < R S  RS 2# )zlinspace(): inferred dtype z& can't be safely cast to passed dtype r3   )default_complex_dtypere   s   r7   rl   ru      s    56K5LLrsxryzr9   zdtype must be torch.dtype, got c                     < R \        S4      P                   R\        S 4      P                   R\        S4      P                   R2# )z4received an invalid combination of arguments - got (, ))type__name__)endstartstepss   r7   rl   ru      sB     u+r$s),,-RU0D0D/EQHr9   zsteps must be IntLike, got c                      R # )z$number of steps must be non-negativer3   r3   r9   r7   rl   ru      s    %Kr9   meta)re   layoutdevice
pin_memoryrequires_grad)ry   r]   r   ro   dimanyrV   r   get_default_dtypeis_complex_dtypere   AssertionErrorr   _check_typer   empty)
r   r   r   basere   r   r   r   r   r   s
   fff&f&&&&@r7   meta_linspace_logspacer   t   s    %&&IIK1P	
 #u||$$GGINP	

 s
CsE/B
Csss
CsE/B
CCC % A A##%!
 =)ELL&&u-z
 22002eU[[))>tE{mLMM 
5'"	H
 eW%%:4;-HII	LL!KL;;	# r9   c                 8  a \         P                  ! SP                  \         P                  8H  V3R  l4       \         P                  ! V P                  4       ^ 8H  ;'       d    SP                  4       ^ 8g  '       * R 4       V P                  SP                  4      # )c                  "   < R S P                    2# )z2take(): Expected a long tensor for index, but got re   indexs   r7   rl   meta_take.<locals>.<lambda>   s    DU[[MRr9   c                      R # )z*take(): tried to take from an empty tensorr3   r3   r9   r7   rl   r      s    <r9   )r]   ro   re   long_check_indexnumel	new_emptyshape)selfr   s   &fr7   	meta_taker      sp     
LLuzz!R
 
ZZ\Q555;;=A#56< >>%++&&r9   r   c                d  a aa S P                   pSP                   p\        P                  ! W48H  R  4       \        P                  ! S P                  S4      ^8H  ;'       d    SP                  S4      ^8H  VVV 3R l4       \	        S P
                  SP
                  4      pS P                  V4      # )c                      R # )z=linalg.cross: inputs must have the same number of dimensions.r3   r3   r9   r7   rl   linalg_cross.<locals>.<lambda>       Or9   c                  V   < R S  RSP                  S 4       RSP                  S 4       2# )zlinalg.cross: inputs dimension z must have length 3. Got  and size)r   otherr   s   r7   rl   r      s1    -cU 399S>"%

3'8:r9   )ndimr]   ro   r   r%   r   r   )r   r   r   x_dy_d	out_shapes   ffd   r7   linalg_crossr      s     ))C
**C	LL
O 
LL		#!44

31 4	
 "$**ekk:I>>)$$r9   c                 ~    \        V R 4       \        V R 4       \        P                  ! V \        P                  R7      # )zlinalg.matrix_expmemory_format)squareCheckInputscheckFloatingOrComplexr]   
empty_likecontiguous_formatr   s   &r7   linalg_matrix_expr      s3     d/04!45D0G0GHHr9   valuesindicesc                 b   \         P                  ! V P                  V P                  V P                  R 7      p\         P                  ! V P                  V P                  \         P
                  R 7      pV P                  4       ^ 8w  d'   V P                  ^ 8w  d   \        WP                  4       W#3# )r   re   )	r]   r   r   r   re   int64r   r   maybe_wrap_dim)r   r   r   r   s   &&  r7   	cummaxminr      sl    
 [[DKKtzzJFkk$**T[[LGzz|qTYY!^sII&?r9   c                 x    \        WP                  4       \        P                  ! V \        P                  R 7      # r   )r   r   r]   r   r   )r   r   s   &&r7   logcumsumexpr      s)     3		"D0G0GHHr9   c                  a VP                   p\        V4      pWV,
          p\        \        V4      4      p\        V4       U	u. uF  p	R NK  	  p
p	V F  pRW&   K	  	  V Uu. uF  qV,          '       d   K  VNK  	  ppV\        V4      ,           p\        V4      pVP	                  4       oVRV pVP                  V3R lRR7       WVR ,           pVP                  V4      pR.\        VP                  VR 4      ,           pVP                  V4      pVP                  ^ 4      pVV^ &   \        V4      p\        \        V4      4       F  pW#V,          ,          VV^,           &   K  	  V P                  V\        P                  R7       \        V4       U	u. uF  p	^ NK  	  pp	^pV^,
          pV^ 8  dD   VV P	                  ^ 4      ,          VVV,          &   VW(V,          ,          ,          pV^,          pKJ  \        Wu4       F,  pV P	                  ^VV,
          ,           4      VVV,          &   K.  	  V P                  VVV P                  4       4       V # u up	i u upi u up	i )FTNc                    < SV ,          # rH   r3   )r<   self_stridess   &r7   rl   _exec_fft.<locals>.<lambda>	  s	    <?r9   keyreverser   )r   lenlistrangestridesortpermuter   reshaper   resize_r]   r   as_strided_storage_offset)outr   	out_sizesr   forwardr   signal_ndim
batch_dimsdim_permuterY   is_transformed_dimdleft	batch_endtmpinputbatched_sizes
batch_sizebatched_out_sizesiout_stridesbatch_numelr   s   &&&&$                 @r7   	_exec_fftr      s$   99Dc(K#J uT{#K).t5A%5 $  #@{!Q*?*?AA{D@c"KD	I;;=L
jy
!CHH*DH9IJ//KLL%E D4JK 899MMM-(EAJ!M!]+3s8_#,V#4!a%  KK!1H1HKI $Dk*k1kK*KQA
q&&1CJJqM&AKN#yQ00	Q:$&)jja*n1E&FKN# %OOI{C,>,>,@AJM 6
 A, +s   I I5II c                R    V ^8  d   QhR\         R\        \        ,          R\        /# )rA   r   r   exclude_last)r   r   intbool)rC   s   "r7   rD   rD   '  s%      V $s) 4 r9   c                    a \        V4      pV P                  4       o\        V4      \        V4      ,
          p\	        VR V V3R lRR7      VR V% V# )Nc                    < SV ,          # rH   r3   )r   r   s   &r7   rl   _sort_dims.<locals>.<lambda>,  s	    ar9   Tr   )r   r   r   r   sorted)r   r   r   sorted_dimsr   r   s   &&&  @r7   
_sort_dimsr   '  sS    s)K;;=L
k
S.
.CDS8$K r9   c                 2  a
 \         P                  ! V P                  P                  4       V'       g   V P	                  4       # \        V 4      R 8X  dN   \         P                  P                  P                  4       '       g    V P                  V P                  4       4      # V P                  4       pV P                  V4      p\        V 4      R8w  d   \        W4      p\        WPWFVR7      # \        V4      pT p VP                  4       o
VP                  V
3R lRR7       \!        \"        \%        V4      4      pV\%        V4      V,
          R p	\        WWWIVR7       VR\%        V4      V,
           pV'       g   V# WpJ d   TpV P                  V4      pK  YuruK  )cpucudar   Tc                    < SV ,          # rH   r3   r   stridess   &r7   rl   meta_fft_c2c.<locals>.<lambda>I  s	    wqzr9   r   N)r]   ro   re   
is_complexclonedevice_hintbackendsmklis_availabler   r   r   r   r   r   r   mincufft_max_ndimr   )r   r   normalizationr   r   outputr   working_tensormax_dims	last_dimsr   s   &&&&      @r7   meta_fft_c2cr  3  sI    
LL&&'zz|4E!%..*<*<*I*I*K*K~~diik**		I^^I&F4F" +ywOO s)KN
 '')14@~s;'78K 08 ; =>	&)P!"?C$4x$?@M!#N^^I.F%3Nr9   c                :    V ^8  d   QhR\         \        ,          /# )rA   r   )r   r   )rC   s   "r7   rD   rD   ]  s      $s) r9   c                     \        V 4      \        8  g-   \        V 4      ^8  d   V ^ ,          ^ 8X  d   V ^,          ^8X  d   R# R# )rA   FT)r   r  r   s   &r7   use_optimized_cufft_pathr  ]  s3    
3x. SX]s1v{s1vQR{r9   c                   a \         P                  ! V P                  P                  4       \	        V P                  4       4      p\	        V4      pVR,          pWF,          ^,          ^,           p\	        V4      pWxV&   V'       d   WuV&   \        V 4      R8X  g   \        V 4      R8X  Ed   V P                  V\        P                  ! V P                  4      R7      p	T p
\        V 4      R8X  d!   \        V4      '       d   \        WWQRR7       M\        V4      ^8X  d   TMTp\        WW.RR7       \        V4      ^8  d2   V P                  V\        P                  ! V P                  4      R7      p
VRR pV'       d}   YrV
P                  4       oVP                  V3R lRR7       \        \         \        V4      4      pV\        V4      V,
          R p\        WWRR7       VR\        V4      V,
           pK  V'       g@   V	P                  V4      WV,          8w  d$   V
P#                  V\         P$                  R	7       T
p	V	# \         P&                  P(                  P+                  4       '       dM   \-        WRR
7      pV P                  V\        P                  ! V P                  4      R7      p	\        WW\RR7      # V P                  V\        P                  ! V P                  4      R7      # )r2   r   xpur   Tr   Nc                    < SV ,          # rH   r3   r   s   &r7   rl   meta_fft_r2c.<locals>.<lambda>  s	    '!*r9   r   r   )r   r   )r]   ro   re   is_floating_pointr   r   r   r   rV   r   r  r   r   r   r   r  r  r   r   r  r  r  r   )r   r   r  onesidedinput_sizesr   last_dimlast_dim_halfsizeonesided_sizesr  r  target_sizesr   r	  r
  r   s   &&&&           @r7   meta_fft_r2cr  d  s    
LL--.tyy{#K[!I2wH#-2Q6+&N08/(4F"k$&75&@ U>>tzzJ   
 t&+CC+H+HfidK ),CA9>LflJPTU3x!|!%U%F%Ftzz%R "0 "
 cr(K)7(//1  ,d !  ~s;/?@'K(88(C(EF	Nt **GC,<x,GH{{8$	(;;&&y@W@W&X'				(	(	*	* >U>>tzzJ   
 ytLL ~~U>>tzzJ  
 	
r9   	generatorc                D    \        V\        P                  ! V .4      4      # rH   )r!   r]   Size)nr  r   s   &$$r7   meta_randpermr     s    S%**aS/22r9   re   r   r   r   c                4    \         P                  ! WW#VR 7      # re   r   r   r   r]   r   )r  re   r   r   r   s   &$$$$r7   meta_randperm_defaultr%    s     ;;	v r9   c                z   a a ^ o\         P                  ! S S8  V V3R l4       \         P                  ! WW4VR7      # )r   c                     < R S RS  2# z:random_ expects 'from' to be less than 'to', but got from=z >= to=r3   highlows   r7   rl   meta_randint.<locals>.<lambda>      LSEQXY]X^_r9   r#  r]   ro   r   )r*  r   re   r   r   r   r+  s   f&$$$$@r7   meta_randintr/    s:     C	LLs
_ ;;&J r9   c                v   a a \         P                  ! SS 8  VV 3R  l4       \         P                  ! W#WEVR7      # )c                     < R S RS  2# r(  r3   r)  s   r7   rl   "meta_randint_low.<locals>.<lambda>  r-  r9   r#  r.  )r+  r*  r   re   r   r   r   s   ff&$$$$r7   meta_randint_lowr3    s5     
LLs
_ ;;&J r9   c                4    \         P                  ! WW#VR 7      # r"  r$  )r   re   r   r   r   s   &$$$$r7   meta_rand_defaultr5    s     ;;&J r9   c                   a a \         P                  ! S P                  4       ^8  ;'       d    S P                  R,          ^8H  V 3R l4       \         P                  ! S P                  \         P
                  8H  V 3R l4       \         P                  ! S^ 8  V3R l4       S P                  RR pS P                  S.VO^N54      # )r2   c                  "   < R S P                    2# )z>_philox_key_split: key must have shape (*batch, 2), got shape r   r   s   r7   rl   'meta_philox_key_split.<locals>.<lambda>  s    PQTQZQZP[\r9   c                  "   < R S P                    2# )z3_philox_key_split: key must have dtype uint64, got r   r9  s   r7   rl   r:    s    Ecii[Qr9   c                     < R S  2# )z4_philox_key_split: num_splits must be positive, got r3   )
num_splitss   r7   rl   r:    s    FzlSr9   Nr   )r]   ro   r   r   re   uint64r   )r   r=  batch_sizess   ff r7   meta_philox_key_splitr@    s    	LL	Q--399R=A-\ 
LL		U\\!Q 
LLQS ))CR.K==*6{6A677r9   c                 0  a  \         P                  ! S P                  4       ^8  ;'       d    S P                  R,          ^8H  V 3R l4       \         P                  ! S P                  \         P
                  8H  V 3R l4       \         P                  ! S 4      # )r2   c                  "   < R S P                    2# )z@_philox_key_fold_in: key must have shape (*batch, 2), got shape r8  r9  s   r7   rl   )meta_philox_key_fold_in.<locals>.<lambda>  s    RSVS\S\R]^r9   c                  "   < R S P                    2# )z5_philox_key_fold_in: key must have dtype uint64, got r   r9  s   r7   rl   rC    s    G		{Sr9   r   )r]   ro   r   r   re   r>  r   )r   datas   f&r7   meta_philox_key_fold_inrF     sh    	LL	Q--399R=A-^ 
LL		U\\!S C  r9   c           	        a aaa \         P                  ! SP                  P                  V V3R  l4       \         P                  ! SP                  \         P                  8H  VV 3R l4       \         P                  ! SP
                  SP
                  8H  VV V3R l4       \         P                  ! SP                  4       ^8  ;'       d    SP                  R
,          ^8H  VV 3R l4       SP                  4       ^8  d   \         P                  ! SP                  4       SP                  4       ^,           8H  VV V3R l4       SP                  RSP                  4        o\         P                  ! \        ;QJ d0    R \        SSP                  4       4       F  '       d   K   RM%	  RM!! R \        SSP                  4       4       4      VV V3R	 l4       R# R# )c                  &   < S  R SP                    2# )z,: self must be a floating point tensor, got r   )op_namer   s   r7   rl   1_check_philox_distribution_args.<locals>.<lambda>  s    7)G

|Tr9   c                  &   < S R S P                    2# )z": key must have dtype uint64, got r   r   rI  s   r7   rl   rJ    s    7)=cii[Ir9   c                  @   < S R SP                    RS P                    2# )z/: self and key must be on the same device, got r   r   r   rI  r   s   r7   rl   rJ    s%    i ;;-uSZZL2r9   c                  &   < S R S P                    2# )z5: key must have shape (2,) or (*batch, 2), got shape r8  rL  s   r7   rl   rJ    s    iLSYYKXr9   c                  @   < S R S P                    RSP                    2# )z?: batched key must have ndim == output ndim + 1, got key shape z with output shape r8  rO  s   r7   rl   rJ  &  s'    ) !!$+>tzzlLr9   Nc              3   H   "   T F  w  rV^8H  ;'       g    W8H  x  K  	  R# 5ir2   Nr3   )r{   kssss   &  r7   r}   2_check_philox_distribution_args.<locals>.<genexpr>-  s%     M2La##28#2Ls   ""FTc                  >   < S R \        S 4       RSP                   2# )z: key batch shape z( is not broadcastable with output shape r   r   )	key_batchrI  r   s   r7   rl   rJ  .  s(    )-d9o-> ?::>**Gr9   r   )
r]   ro   re   r  r>  r   r   r   allzip)rI  r   r   rY  s   fff@r7   _check_philox_distribution_argsr\    s2   	LL

$$T 
LL		U\\!I 
LLszz!	
 
LL	Q--399R=A-	
 wwy1}GGIa'	
 IIl
+	CM#i2LMCCCM#i2LMM	
 r9   c                     \        R W4       V # )_philox_normal_r\  )r   r   meanstds   &&&&r7   meta_philox_normal_rb  5  s    #$5tAKr9   c                     \        R W4       V # )_philox_uniform_r_  )r   r   r+  r*  s   &&&&r7   meta_philox_uniform_re  ;  s    #$6BKr9   c                ^    V ^8  d   QhR\         R\        \        ,          R\        R\        /# )rA   r   r   r  lastdim)r   r   r   )rC   s   "r7   rD   rD   C  s2     *L *Lv *LDI *Lc *LC *Lr9   c                 ,   \         P                  ! V P                  P                  4       \	        V 4      R 8X  d   \        V P                  4       4      pW4VR,          &   V P                  V\        V P                  4      R7      p\        V4      '       d/   \        VV P                  \         P                  R7      VVRR7      # \        V4      ^8  d   \        WRR ^ RR7      pM V P                  \         P                  R7      p\        WVWAR,          .RR7      # \         P                  P                   P#                  4       '       d   T p\        V4      ^8  d   VRR p\        WVRR7      pVRR p\        VP                  4       4      pW4VR,          &   V P                  V\        V P                  4      R7      p	\        WWARR7      # \        V P                  4       4      pW4VR,          &   V P                  V\        V P                  4      R7      # )r   r   r   Fr   Nr   )r]   ro   re   r   r   r   r   r   rg   r  r   r   r   r   r  r  r  r  )
r   r   r  rg  r   r  tempr   c2c_dimsr   s
   &&&&      r7   meta_fft_c2rrk  A  s    
LL&&'4F"%	$#b'	1LM#C((

)@)@
A  3x!|#Dcr(AuEzz0G0GzHV92wiOO				(	(	*	*s8a<3BxH NEbc(C&	$#b'nnYodjj.InJYUCC %	$#b'~~itzz/J~KKr9   c                 "   ^ RI Hp V! V 4      '       g'   \        P                  ! V 4      ^8X  d   \	        R4      h\        V\        4      '       d?   VP                  W4      p\        P                  P                  W@P                  4       4       V # )r   free_unbacked_symbolszQmore than one element of the written-to tensor refers to a single memory location)%torch.fx.experimental.symbolic_shapesrn  r]   _debug_has_internal_overlapRuntimeErrorry   r   to_refsexpandr   )r   srcnon_blockingrn  intermediates   &&&  r7   
meta_copy_rx  p  st     L "$''E,M,Md,SWX,X_
 	
 #vvvd1 	<5Kr9   c                    \        V P                  4       4      p\        V P                  4       4      pWP                  4       8  d   ^MW!,          W1,          ,          pVP	                  V^4       VP	                  W4       W#3# r1   )r   r   r   r   insert)tensorr   result_sizesresult_strides
new_strides   &&   r7   inferUnsqueezeGeometryr    sf    &L&--/*NZZ\)|/@>CV/VJQ#*''r9   c                     \        WP                  4       ^,           4      p\        W4      w  r#V P                  W#4       V # r1   )r   r   r  r   )r   r   g_sizes	g_stridess   &&  r7   meta_unsqueeze_r    s6    
hhj1n
-C/:GW(Kr9   c                    V ^8  d   QhR\         R\         R\         R\         R,          R\        R,          R\        P                  R,          /# )rA   r   weight_metabiasN_activation_opt	out_dtype)r   strr]   re   )rC   s   "r7   rD   rD     sU     ) ))) ) 4-	)
 4Z) {{T!)r9   c                 X   \        V P                  4      pVeU   VP                  ^ 4      VP                  ^ 4      8w  d0   \        RVP                  ^ 4       RVP                  ^ 4       24      hVP                  ^4      V P                  R
4      ^,          8w  d7   \        RVP                  ^4       RV P                  R
4      ^,           24      hVP                  ^ 4      VR
&   \	        V P                  4      ^8w  d#   \        R\	        V P                  4       R24      h^V P                  ^ 4      3pVeP   V P
                  \        P                  8X  d   V\        P                  8X  g   \        RV P
                   RV 24      hT P                  TTVf   V P
                  MTR	7      pV# )Nz%output size mismatch: weight.size(0)= != bias.size(0)=zweight.size(1)=z != input.size(-1)/2=z0we can only handle the squashed input case, got D inputzKout_dtype is only supported for i8i8->i32 linear operator, got input.dtype=, out_dtype=r   r   )
r   r   r   r   r   re   r]   int8int32new_empty_strided)	r   r  r  r  r  r  output_sizestransposed_stridesr  s	   &&&&&&   r7   meta_sparse_structured_linearr    s    $L;;q>TYYq\) 7A7GGXY]YbYbcdYeXfg  {{1~B!++fkk!n--B5::b>TUCUBVW
 	
 {{1~L 5;;1>s5;;?O>PPWX
 	
 UZZ]+uzz)i5;;.F ]^c^i^i]jjv  xA  wB  C  $$&.ekkI % F Mr9   c          	      j    V ^8  d   QhR\         R\         R\         R\        P                  R,          /# )rA   mat1	mat1_metamat2r  Nr   r]   re   )rC   s   "r7   rD   rD     s9      
  {{T!	r9   c                 b   \        V P                  4      ^8w  d#   \        R\        V P                  4       R24      h\        VP                  4      ^8w  d#   \        R\        VP                  4       R24      h\        VP                  4      ^8w  d#   \        R\        VP                  4       R24      hV P                  ^4      VP                  ^ 4      ^,          8w  d7   \        RV P                  ^4       RVP                  ^ 4      ^,           24      hV P                  ^ 4      VP                  ^4      .pVeP   VP                  \
        P                  8X  d   V\
        P                  8X  g   \        RVP                   RV 24      hTP                  TVf   VP                  MTR	7      pV# )
rA   mat1 must be 2D, got Dmat1_meta must be 2D, got mat2 must be 2D, got mat1.size(1)= != mat2.size(0)/2=Jout_dtype is only supported for i8i8->i32 linear operator, got mat2.dtype=r  r   	r   r   r   r   re   r]   r  r  r   )r  r  r  r  r  r  s   &&&&  r7   meta_sparse_structured_mmr    sv    4::!4S_4EQGHH
9??q 9#ioo:N9OqQRR
4::!4S_4EQGHHyy|tyy|a''DIIaL>)<TYYq\A=M<NO
 	
 IIaL$))A,/L

ejj(Y%++-E \]a]g]g\hhtu~t  A  ^^%-djj9  F
 Mr9   alphabetar  c                v    V ^8  d   QhR\         R\         R\         R\         R\        P                  R,          /# )rA   r   r  r  r  r  Nr  )rC   s   "r7   rD   rD     sC     ) ))
) ) 	) {{T!)r9   c                   \        V P                  4      ^8w  d#   \        R\        V P                  4       R24      h\        VP                  4      ^8w  d#   \        R\        VP                  4       R24      h\        VP                  4      ^8w  d#   \        R\        VP                  4       R24      h\        VP                  4      ^8w  d#   \        R\        VP                  4       R24      hV P                  ^ 4      VP                  ^ 4      8w  d0   \        RV P                  ^ 4       RVP                  ^ 4       24      hVP                  ^4      VP                  ^ 4      ^,          8w  d7   \        R	VP                  ^4       R
VP                  ^ 4      ^,           24      hVP                  ^ 4      VP                  ^4      .pVeP   VP                  \
        P                  8X  d   V\
        P                  8X  g   \        RVP                   RV 24      hTP                  TVf   VP                  MTR7      pV# )r2   zKonly input broadcasted to columns of mat1 * mat2 product is supported, got r  r  r  r  r  zUonly input broadcasted to columns of mat1 * mat2 product is supported, input.size(0)=z != mat1.size(0)=r  r  r  r  r   r  )	r   r  r  r  r  r  r  r  r  s	   &&&&$$$  r7   meta_sparse_structured_addmmr    s    5;;1YZ]^c^i^iZjYkkrs
 	
 4::!4S_4EQGHH
9??q 9#ioo:N9OqQRR
4::!4S_4EQGHHzz!}		!$"ZZ]O+<TYYq\NL
 	
 yy|tyy|a''DIIaL>)<TYYq\A=M<NO
 	
 IIaL$))A,/L

ejj(Y%++-E \]a]g]g\hhtu~t  A  ^^%-djj9  F
 Mr9   c                    V ^8  d   QhR\         P                  R\         P                  R\        R,          R\        R,          R\         P                  R,          R\        R\        R	\        R
\        /	# )rA   compressed_Adense_Br  Nr  r  transpose_resultalg_idsplit_ksplit_k_mode)r]   r   re   r   r   )rC   s   "r7   rD   rD     s{     ;< ;<,,;<\\;< 4-;< D=	;<
 {{T!;< ;< ;< ;< ;<r9   c	           	      t   VP                   \        P                  \        P                  \        P                  \        P
                  \        P                  09  d   \        R VP                    24      hV P                   VP                   8w  d&   \        RV P                    RVP                    24      h\        VP                  4      ^8w  d#   \        R\        VP                  4       R24      hV P                   \        P
                  \        P                  39   p	V	'       d"   VP                  4       '       d   \        R4      hVP                  ^4      p
V P                  ^ 4      pVe6   WP                  ^ 4      8w  d!   \        RV RVP                  ^ 4       24      hVeu   V	'       dD   V\        P                  \        P                  \        P                  \        P                  09   g*   \        RV P                    R	VP                    R
V R24      hV'       d   W3MW3pVP                  WR7      # )zA_cslt_sparse_mm only supports fp16, bf16, int8, and fp8e4m3, got z%inputs must have the same dtype, got r   z-_cslt_sparse_mm only supports 2d inputs, got r  z.dense input must be transposed for 8bit dtypeszbias size mismatch: m=r  zout_dtype is not supported for z x z -> z matmul!r   )re   r]   float32float16bfloat16r  float8_e4m3fnr   r   r   is_contiguousr   r  r   )r  r  r  r  r  r  r  r  r  is_8bit_input_typer  moutput_shapes   &&&&&&&&&    r7   meta__cslt_sparse_mmr    s    }}

  OPWP]P]_
 	
 W]]*3L4F4F3GuW]]O\
 	
 7==Q;C<N;OqQ
 	
 &++

E<O<O/PP  "" !QRRQA!A		! (+<TYYq\NK  ##	 !1,2D2D1ESW[\e[ffno  .A6A6L\;;r9   include_selfTc                    V ^8  d   QhR\         R\        R\         R\        P                   R\        R\        R\         /# rA   r   r   r   sourcereducer  rB   r   r   r]   r  r   )rC   s   "r7   rD   rD   \  s[     	I 	I
	I		I 	I LL		I
 	I 	I 	Ir9   c                N    \         P                  ! V \         P                  R 7      # r   r]   r   r   r   r   r   r  r  r  s   &&&&&$r7   meta_index_reducer  [  s     D0G0GHHr9   c                    V ^8  d   QhR\         R\        R\         R\        P                   R\        R\        R\         /# r  r  )rC   s   "r7   rD   rD   i  sR     	 	
			 	 LL		
 	 	 	r9   c                    V # rH   r3   r  s   &&&&&$r7   meta_index_reduce_r  h  s	     Kr9   c                     \        V P                  4       4      pV P                  4       ^ 8  d   VP                  4       W1&   V P	                  V4      # r   )r   r   r   r   r   )r   r   r   result_sizes   &&& r7   meta_index_selectr  v  s>     tyy{#KxxzA~ ;;=>>+&&r9   lengthsoffsetsaxisunsafeinitialc                    V ^8  d   QhR\         R\        R\         R,          R\         R,          R\         R,          R\        R\        R	\         /# )
rA   rE  r  r  Nr   r  r  r  rB   )r   r  r   r   )rC   s   "r7   rD   rD     sn      W  W
 W W d]	 W
 d] W d] W  W  W  Wr9   c                   a a Ve   \        R4      hVV 3R lpVe   V! VP                  4      # Ve8   VP                  R R VP                  R,          ^,
          3,           p	V! V	4      # \        R4      h)Nz?segment_reduce(): indices based reduction is not supported yet.c                    < \         P                  ! V SP                  S^,           R ,           SP                  R\         P                  R7      # )r2   Nr   re   r   r   )r]   r   r   re   r   )lengths_shaper  rE  s   &r7   segment_reduce_lengths_tensor:meta_segment_reduce.<locals>.segment_reduce_lengths_tensor  s>    {{DJJtaxz22**11	
 	
r9   z<segment_reduce(): Either lengths or offsets must be defined.r   )NotImplementedErrorr   rq  )
rE  r  r  r   r  r  r  r  r  r  s
   f&$$$d$$  r7   meta_segment_reducer    s|     !M
 	

 ,W]];; cr*gmmB.?!.C-EE,];;
U
VVr9   c                 $    V P                  R4      # Nr3   r   r   s   &r7   meta_maxr         >>"r9   c                     \         P                  ! V P                  V34      p\        WV4      pV P	                  V4      V P	                  V\
        P                  R 7      3# r   rV   reduction_dimsr   _compute_reduction_shaper   r]   r   r   r   keepdimr  s   &&& r7   meta_max_dimr    P    


tzzC6
2C+Dw?L|$|5::6 r9   c                 $    V P                  R4      # r  r  r   s   &r7   meta_minr    r  r9   c                     \         P                  ! V P                  V34      p\        WV4      pV P	                  V4      V P	                  V\
        P                  R 7      3# r  r  r  s   &&& r7   meta_min_dimr    r  r9   c                     V P                  4       '       d   \        V P                  4      pM\        V \        P
                  R 7      w  r!\        P                  ! WR7      # rU   r   )r   r   re   r   r   INT_TO_FLOATr]   r   )r   rZ   rY   s   &  r7   
meta_angler    sH    /

;, ? L L
 D55r9   c                     \         P                  ! WP                  4       V P                  4       VP	                  \         P
                  ! V 4      4      # rH   )r]   _resize_output_r   r   copy_angle)r   r   s   &&r7   meta_angle_outr    s4    	#yy{DKK899U[[&''r9   c                     R # rH   r3   )vals   &r7   assert_asyncr        
r9   c                     R # rH   r3   )r  
assert_msgs   &&r7   assert_async_metar    r  r9   c                     R # rH   r3   )ss   &r7   
print_metar    r  r9   r   c                 2    \         P                  ! RR R7      # )r   rN  r3   r$  re   r   r   r   r   s   $$$$$r7   make_dep_tokenr    s     ;;r&))r9   c                 t    ^ RI Hp \        V \        \        34      '       d   \        R4      hV! WVR7       R# )r   )constrain_range'Constraining SymFloat or Symbool is nyir  maxN)ro  r  ry   r
   r	   
ValueError)r   r  r  r  s   &&& r7   sym_constrain_ranger    s0     F$7+,,BCCDs+r9   c                 4    \         P                  WVR 7       V# r  )r-   r  r   r  r  	dep_tokens   &&&&r7   functional_sym_constrain_ranger    s    T4r9   c                 T   ^ RI Hp Vf    Vf   \        P                  ! V ^ 8  4       R# \	        V \
        \        34      '       d   \        R4      h\        V 4      \        J d;   Ve   \        P                  ! W8  4       Ve   \        P                  ! W8*  4       R# V! WVR7       R# )r   )_constrain_range_for_sizeNr  r  )
ro  r  r]   ro   ry   r
   r	   r  r   r   )r   r  r  r  s   &&& r7   sym_constrain_range_for_sizer     s     P
{s{TQY$7+,,BCCDzS?LL%?LL%d5r9   c                 4    \         P                  WVR 7       V# r
  )r-   r  r  s   &&&&r7   'functional_sym_constrain_range_for_sizer    s    %%d%=r9   c                     V# rH   r3   )r  r  r  s   &&&r7   functional_assert_async_metar    s    r9   c                0    V ^8  d   QhR\         R\        /# )rA   r   f_namer   r  )rC   s   "r7   rD   rD      s     
 
F 
C 
r9   c                    a a S P                  4       ^8  d   \        S RS P                  4        24      h\        P                  ! S P	                  R4      S P	                  R4      8H  VV 3R l4       R# )rA   z8: The input tensor must have at least 2 dimensions, got c                  V   < S  R SP                  R4       RSP                  R4       R2# )z5: A must be batches of square matrices, but they are  by 	 matricesr   r   )r  r   s   r7   rl   #squareCheckInputs.<locals>.<lambda>(  s/    6( 		"d499R=/Dr9   Nr   r  )r   r   r]   ro   r   )r   r  s   ffr7   r   r      s^    xxzA~hNtxxzl[
 	
 
LL		"2&	Dr9   c                <    V ^8  d   QhR\         R\         R\        /# )rA   r   Anamer  )rC   s   "r7   rD   rD   0  s!          F  #  r9   c                   a aa \         P                  ! S P                  SP                  8H  VV 3R  l4       \         P                  ! S P                  SP                  8H  VV 3R l4       \         P                  ! SP	                  R4      SP	                  R4      8H  V3R l4       \         P                  ! SP	                  R4      S P	                  R4      8H  VVV 3R l4       R# )c                  >   < R SP                    RS P                    R2# )z:Expected b and A to be on the same device, but found b on z
 and A on 	 instead.rN  r  r   s   r7   rl   (linearSolveCheckInputs.<locals>.<lambda>3  s     H{{m:ahhZy:r9   c                  >   < R SP                    RS P                    R2# )z=Expected b and A to have the same dtype, but found b of type z and A of type r#  r   r$  s   r7   rl   r%  ;  s     Kzzl/!'')=r9   c                  R   < R S P                  R4       RS P                  R4       R2# )z3A must be batches of square matrices, but they are r  r  r  r   r   r  s   r7   rl   r%  C  s+    FF2J<tAFF2J<yBr9   c                     < R S RS P                  R4       RS P                  R4       RSP                  R4       RSP                  R4       2
# )zIncompatible matrix sizes for z: each A matrix is r  z but each b matrix is r   r  r   )r  r   r   s   r7   rl   r%  K  sM    ,TF 3D$TYYr]O4		"Hr9   Nr   r  )r]   ro   r   re   r   )r   r  r   s   fffr7   linearSolveCheckInputsr*  0  s    	LLqxx	
 
LL

agg	
 
LL	r
affRj 	
 
LL	r
diim#	
r9   c                <    V ^8  d   QhR\         R\        R\        /# )rA   tr  allow_low_precision_dtypesr   r  r   )rC   s   "r7   rD   rD   T  s&     
 


 !%
r9   c                 p  aa V P                   o\        P                  ! V P                  4       ;'       g    V P	                  4       VV3R  l4       V'       g^   \        P                  ! S\        P
                  \        P                  \        P                  \        P                  39   VV3R l4       R# R# )c                     < S R S  2# )z<: Expected a floating point or complex tensor as input. Got r3   re   r  s   r7   rl   (checkFloatingOrComplex.<locals>.<lambda>\  s    6(VW\V]^r9   c                     < S R S  2# )z*: Low precision dtypes not supported. Got r3   r1  s   r7   rl   r2  a  s    vhHPr9   N)	re   r]   ro   r  r   ra   rc   r`   rb   )r,  r  r-  re   s   &f&@r7   r   r   T  sq    
 GGE	LL	//^ &ekk5<<u}}MMP	
 &r9   c                <    V ^8  d   QhR\         R\        R\        /# )rA   r  r  arg_namer  )rC   s   "r7   rD   rD   f  s!      V S C r9   c                 d   aa \         P                  ! V P                  4       ^8  VV3R l4       R# )rA   c                     < S R S  R2# )z: The input tensor z! must have at least 2 dimensions.r3   )r5  r  s   r7   rl   checkIsMatrix.<locals>.<lambda>i  s    6(-hZ7XYr9   N)r]   ro   r   )r  r  r5  s   &ffr7   checkIsMatrixr9  f  s    	LL	1Yr9   c                H    V ^8  d   QhR\         R\         R\        R\        /# )rA   r  Br   r  r   r   r  )rC   s   "r7   rD   rD   m  s(     
 
 
F 
$ 
 
r9   c                   a aaa \        S S4       \        SS4       \        P                  ! S'       d$   S P	                  R4      SP	                  R4      8H  M"S P	                  R4      SP	                  R4      8H  V VVV3R l4       R# )rA   c                     < S R S'       d   RMR RS P                  R4       RS P                  R4       RSP                  R4       RSP                  R4       R2# )	z2: Incompatible shapes of A and B for the equation zAX = BzXA = Bz (r<   r   r   r  r   r   )r  r;  r  r   s   r7   rl   #checkInputsSolver.<locals>.<lambda>r  sV    hHxX.AaffRj\qvvbzl!AFF2J<qJr9   Nr  r   )r   r9  r]   ro   r   )r  r;  r   r  s   ffffr7   checkInputsSolverr@  m  sY    a !V	LL$(r
affRj affRjAFF2J.F	
r9   resultc                H    V ^8  d   QhR\         R\        R\        R\         /# )rA   fn_namerA  r   result_name)r  r   )rC   s   "r7   rD   rD   z  s0        	r9   c                 x   a aaa \         P                  ! SP                  SP                  8H  V VVV3R  l4       R# )c            	      L   < S  R S RS RSP                    RSP                    2	# )z: Expected z5 and input tensors to be on the same device, but got z on z and input on rN  )rC  r   rA  rD  s   r7   rl   !checkSameDevice.<locals>.<lambda>  s0    i{;-/dm4nU\\NLr9   N)r]   ro   r   )rC  rA  r   rD  s   ffffr7   checkSameDevicerH  z  s&     
LL%	
r9   c                $    V ^8  d   QhR\         /# )rA   UPLO)r  )rC   s   "r7   rD   rD     s      C r9   c                    a  S P                  4       p\        P                  ! \        S 4      ^8H  ;'       d    VR8H  ;'       g    VR8H  V 3R l4       R# )r2   ULc                     < R S  2# )z1Expected UPLO argument to be 'L' or 'U', but got r3   )rJ  s   r7   rl   checkUplo.<locals>.<lambda>  s    CD6Jr9   N)upperr]   ro   r   )rJ  UPLO_uppercases   f r7   	checkUplorR    sB    ZZ\N	LLD	QKKNc1JJ^s5JJr9   eigenvalueseigenvectorsrM  c                <    V ^8  d   QhR\         R\        R\        /# )rA   r  rJ  	compute_vr.  )rC   s   "r7   rD   rD     s!       s T r9   c                 ^   \        V R 4       \        V4       \        V P                  4      pV'       d0   V P	                  V4      pVP                  V\        VRR7      4       MV P	                  ^ .4      pVP                  4        V P	                  V\        V P                  4      R7      pWT3# )zlinalg.eighF	row_majorr   )
r   rR  r   r   r   r   r   poprg   re   )r  rJ  rV  r   vecsvalss   &&&   r7   meta__linalg_eighr]    s     a'dOME{{5! ;EU ST{{A3	IIK;;uOAGG$<;=D:r9   c                0    V ^8  d   QhR\         R\         /# )rA   r   rB   r   )rC   s   "r7   rD   rD     s     B B B6 Br9   c                     \        V R 4       \        P                  ! V P                  4      '       d   V P                  M\        P                  ! V P                  4      pV P                  V P                  RR VR7      # )zlinalg.eigvalsNr   r   )r   rV   r   re   r   r   r   )r   complex_dtypes   & r7   meta__linalg_eigvalsrb    sf     e-. !!%++.. 	..u{{; 
 ??5;;s+=?AAr9   c                $    V ^8  d   QhR\         /# )rA   r   r_  )rC   s   "r7   rD   rD     s      6 r9   c                    \        V R 4       \        P                  ! V P                  4      '       d   V P                  M\        P                  ! V P                  4      pV P                  V P                  RR VR7      pV P                  V P                  VR7      p\        V 4      R8H  pVP                  V P                  \        V P                  VR7      4       W#3# )z
linalg.eigNr   r   rX  r   )
r   rV   r   re   r   r   r   r   r   r   )r   ra  r   vectorsis_cudas   &    r7   meta_linalg_eigrg    s     e\* !!%++.. 	..u{{; 
 __U[["-]_CFooekko?G% F*G0P ?r9   c                0    V ^8  d   QhR\         R\         /# )rA   ru  rB   r_  )rC   s   "r7   rD   rD     s     Q Q QF Qr9   c                 v    V P                   P                  \        P                  R 7      P	                  RR4      # )r   r  r   )mTr   r]   r   	transpose)ru  s   &r7   cloneBatchedColumnMajorrl    s*    66<<e&=&=<>HHRPPr9   c                H    V ^8  d   QhR\         R\         R\        R\         /# rA   r   r  rP  rB   r   r   )rC   s   "r7   rD   rD     s(     ) ) )F )4 )F )r9   c                     \        V 4      # rH   )rl  )r   r  rP  s   &&&r7   _cholesky_solve_helperrq    s     #4((r9   c                H    V ^8  d   QhR\         R\         R\        R\         /# rn  ro  )rC   s   "r7   rD   rD     s.     J J JF J4 JF Jr9   c                    a a \         P                  ! S P                  ^8  V 3R l4       \         P                  ! SP                  ^8  V3R l4       \        S SR4      w  r4\	        W4V4      # )rA   c                  $   < R S P                    R2# )z-b should have at least 2 dimensions, but has  dimensions insteadr   r   s   r7   rl    cholesky_solve.<locals>.<lambda>  s    ?		{J]^r9   c                  $   < R S P                    R2# )z-u should have at least 2 dimensions, but has ru  rv  r(  s   r7   rl   rw    s    ?xGZ[r9   cholesky_solve)r]   ro   r   !_linalg_broadcast_batch_dims_namerq  )r   r  rP  self_broadcastedA_broadcasteds   ff&  r7   ry  ry    sd     
LL		Q^ 
LL	![ 'Ha!'# ""25IIr9   c                <    V ^8  d   QhR\         R\        R\         /# rA   r   rP  rB   ro  )rC   s   "r7   rD   rD     s!     ) )6 )$ )6 )r9   c                     V P                  4       ^ 8X  d'   \        P                  ! V \        P                  R7      # \	        V R4       \        V 4      # )r   r   cholesky)r   r]   r   legacy_contiguous_formatr   rl  r   rP  s   &&r7   r  r    s@     zz|qE4R4RSSdJ'"4((r9   c                <    V ^8  d   QhR\         R\        R\         /# r~  ro  )rC   s   "r7   rD   rD     s!     ) )6 )$ )6 )r9   c                 0    \        V R 4       \        V 4      # )cholesky_inverse)r   rl  r  s   &&r7   r  r    s     d./"4((r9   c                <    V ^8  d   QhR\         R\        R\        /# )rA   r  rP  check_errorsro  )rC   s   "r7   rD   rD     s!      &  T r9   c                    \        V R 4       \        V R 4       V P                  p\        V4      p\	        VR4      pV P                  V4      pVP                  W54       V P                  V^ V^,
           \        P                  R7      pWg3# )zlinalg.choleskyFr   )	r   r   r   r   r   r   r   r]   r  )r  rP  r  A_shaper   	L_stridesrM  infoss   &&&     r7   linalg_cholesky_exr    s|    a*+1/0ggGw<D ,GU;I	GAMM'% KKD1H-U[[KAE8Or9   c                <    V ^8  d   QhR\         R\         R\         /# )rA   r   taurB   r_  )rC   s   "r7   rD   rD     s!     . .f .6 .f .r9   c                 `  a aa \         P                  ! S P                  ^8  R 4       \         P                  ! S P                  R4      S P                  R4      8  R 4       \         P                  ! S P                  R4      SP                  R4      8  R 4       \         P                  ! S P                  SP                  ,
          ^8H  V V3R l4       S P                  ^8  d=   S P                  RR pSP                  RR o\         P                  ! SV8H  V3R l4       \         P                  ! SP
                  S P
                  8H  V V3R l4       \        RSS R	4       \         P                  ! S P                  \        S P                  R
R7      S P
                  S P                  R7      # )rA   c                      R # )zHtorch.linalg.householder_product: input must have at least 2 dimensions.r3   r3   r9   r7   rl   ,linalg_householder_product.<locals>.<lambda>      Zr9   c                      R # )zbtorch.linalg.householder_product: input.shape[-2] must be greater than or equal to input.shape[-1]r3   r3   r9   r7   rl   r    s    tr9   c                      R # )z`torch.linalg.householder_product: input.shape[-1] must be greater than or equal to tau.shape[-1]r3   r3   r9   r7   rl   r    s    rr9   c                  <   < R SP                    RS P                    2# )zptorch.linalg.householder_product: Expected tau to have one dimension less than input, but got tau.ndim equal to  and input.ndim is equal to rv  r   r  s   r7   rl   r    "    )),
2Nuzzl\r9   Nc                     < R S  2# )zltorch.linalg.householder_product: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r3   actual_batch_tau_shapes   r7   rl   r        66L5MOr9   c                  <   < R SP                    RS P                    2# )z,torch.linalg.householder_product: tau dtype z does not match input dtype r   r  s   r7   rl   r  &  s    :399+*5;;-9r9   z torch.linalg.householder_productr  FrX  r   r   re   r   r  r   )
r]   ro   r   r   r   re   rH  empty_stridedr   r   )r   r  expected_batch_tau_shaper  s   ff @r7   linalg_householder_productr     sK   
 
LL

aZ 
LL

2%**R.(t 
LL

2#((2,&r
 
LL

SXX"	
 zzA~#(;;s#3 !$3B"&>>	
 
LL		U[[ 	
 6UEJ[[*5;;%Hkk||	 r9   c                0    V ^8  d   QhR\         R\        /# )rA   r  r  ro  )rC   s   "r7   rD   rD   7  s      &  r9   c                 0   \        V R 4       \        V R RR7       V P                  V P                  4      pVP	                  V P                  \        V P                  RR7      4       V P                  V P                  RR \        P                  R7      pW#3# )zlinalg.inv_exF)r-  rX  Nr   r  r   r   r   r   r   r   r]   r  )r  r  rM  r  s   &&  r7   linalg_inv_ex_metar  6  so    a)1o%P	AGGAMM!''6qww%PQKKEKKK8E8Or9   LDpivotsinfo	hermitianr  c                t    V ^8  d   QhR\         R\        R\        R\        \         \         \         3,          /# )rA   r   r  r  rB   r   r   rn   )rC   s   "r7   rD   rD   D  s>      
  	
 666!"r9   c                   \        V R 4       \        V R 4       \        P                  ! V P                  \        V P                  RR7      V P                  V P                  R7      pV P                  V P                  RR \        P                  R7      pV P                  V P                  RR \        P                  R7      pW4V3# )ztorch.linalg.ldl_factor_exFrX  r  Nr   r   r  )
r   r   r]   r  r   r   re   r   r   r   )r   r  r  r  r  r  s   &$$   r7   linalg_ldl_factor_ex_metar  B  s     d894!=>			ZZ*4::Gjj{{	
B ^^DJJsO599^=F>>$**Sb/>;Dtr9   c          
      T    V ^8  d   QhR\         R\         R\         R\        R\         /# )rA   r  r  r;  r  rB   ro  )rC   s   "r7   rD   rD   Y  s:     ' ''' '
 ' 'r9   c                v  a aa \        S R 4       \        S R 4       \        SS R 4       \        P                  ! SP
                  ^8  V3R l4       S P                  RR	 p\        P                  ! VSP                  8H  V3R l4       \        P                  ! \        P                  ! SP                  4      V3R l4       \        P                  ! S P                  SP                  8H  VV 3R l4       \        SS 4      w  rV\        P                  ! V\        VRR7      SP                  SP                  R7      # )
ztorch.linalg.ldl_solvec                  $   < R S P                    R2# )zMtorch.linalg.ldl_solve: Expected B to have at least 2 dimensions, but it has ru  rv  )r;  s   r7   rl   'linalg_ldl_solve_meta.<locals>.<lambda>e      &&!46r9   Nc                  $   < R S P                    R2# )zjtorch.linalg.ldl_solve: Expected LD.shape[:-1] and pivots.shape to be the same, but got pivots with shape  insteadr8  r  s   r7   rl   r  m      ))/h@r9   c                  "   < R S P                    2# )z<torch.linalg.ldl_solve: Expected pivots to be integers. Got r   r  s   r7   rl   r  t  s    Nv||n]r9   c                  <   < R SP                    RS P                    2# )z!torch.linalg.ldl_solve: LD dtype z does not match b dtype r   )r;  r  s   r7   rl   r  x  s     3BHH:=UVWV]V]U^_r9   FrX  r  r   )r   r   r*  r]   ro   r   r   rV   is_integer_dtypere   _linalg_broadcast_batch_dimsr  r   r   )r  r  r;  r  expected_pivots_shapeB_broadcast_sizerY   s   fff$   r7   linalg_ldl_solve_metar  W  s     b232781b":;	LL	!	
 HHSbM	LL-	
 
LLv||,] 
LL
AGG_ 7q"=*+;uMggxx	 r9   PrL  pivotc          	      h    V ^8  d   QhR\         R\        R\        \         \         \         3,          /# )rA   r  r  rB   r  )rC   s   "r7   rD   rD     s-      f  fff>T8U r9   c                |  a  \         P                  ! S P                  ^8  V 3R l4       \        S P                  4      pVR,          pVR,          p\        W44      pW2R&   V'       d   S P                  V4      pMS P                  ^ .4      pWRR&   S P                  V4      pWRR&   WBR&   S P                  V4      pWgV3# )rA   c                  $   < R S P                    R2# )z@linalg.lu: Expected tensor with 2 or more dimensions. Got size: r  r8  r(  s   r7   rl    linalg_lu_meta.<locals>.<lambda>  s    RSTSZSZR[[cdr9   r  r   )r]   ro   r   r   r   r   r   )	r  r  sizesr  r  kr  rM  rL  s	   f$       r7   linalg_lu_metar    s     
LL	!d
 MEb	Ab	AA"IKKKK"I	EA"I"I	EA7Nr9   LUc                t    V ^8  d   QhR\         R\        R\        R\        \         \         \         3,          /# )rA   r  r  r  rB   r  )rC   s   "r7   rD   rD     s>            	 
 666!" r9   c                  a  \         P                  ! S P                  ^8  V 3R l4       \        S P                  4      pVR,          pVR,          p\         P
                  ! V\        VRR7      S P                  S P                  R7      pVP                  4        \        WE4      VR&   S P                  V\         P                  R7      pVP                  4        S P                  V\         P                  R7      pWgV3# )rA   c                  $   < R S P                    R2# )zFtorch.lu_factor: Expected tensor with 2 or more dimensions. Got size: r  r8  r(  s   r7   rl   *linalg_lu_factor_ex_meta.<locals>.<lambda>  s    XYZY`Y`Xaaijr9   FrX  r  r   r  r   )r]   ro   r   r   r   r  r   re   r   rZ  r   r   r   )	r  r  r  r  r  r  r  r  r  s	   f$$      r7   linalg_lu_factor_ex_metar    s     
LL	!j
 MEb	Ab	A			*5EBggxx	
B 
IIKE"I[[eii[0F 
IIK;;uEII;.Dtr9   r   adjointc                `    V ^8  d   QhR\         R\         R\         R\        R\        R\         /# )rA   r  r  r;  r   r  rB   ro  )rC   s   "r7   rD   rD     sD     4 444 4
 4 4 4r9   c                  a aa \        S R 4       \        P                  ! S P                  SP                  8H  VV 3R l4       \        P                  ! SP                  \        P                  8H  R 4       \        S R 4       \        S SVR4       \        P                  ! S P                  R	4      SP                  R	4      8H  R 4       \        P                  ! S P                  RR	 SP                  8H  V3R l4       \        SS 4      w  rV\        P                  ! V\        WS'       * R7      SP                  SP                  R7      pVP                  4       ^ 8w  d/   V'       g'   VP                  4       '       d   VP                  4       pV# )
ztorch.linalg.lu_solvec                  >   < R SP                    RS P                    R2# )zPlinalg.lu_solve: Expected LU and B to have the same dtype, but found LU of type  and B of type r  r   )r;  r  s   r7   rl   &linalg_lu_solve_meta.<locals>.<lambda>  s#    $$&HH:_QWWIXOr9   c                      R # )zElinalg.lu_solve: pivots should be a Tensor of scalar type torch.int32r3   r3   r9   r7   rl   r    s    Wr9   zlinalg.lu_solvec                      R # )zYlinalg.lu_solve: Number of pivots per batch should be same as the dimension of the matrixr3   r3   r9   r7   rl   r    s    kr9   Nc                  $   < R S P                    R2# )zclinalg.lu_solve: Expected LU.shape[:-1] and pivots.shape to be the same, but got pivots with shape r  r8  r  s   r7   rl   r    r  r9   rX  r  r   )r   r]   ro   re   r   r   r@  r   r   r  r  r   r   r   r   conj)r  r  r;  r   r  r  rY   rA  s   fff$$   r7   linalg_lu_solve_metar    s-    267	LL
AGG	
 
LL		!W b12b!T#45	LL
v{{2&k 
LL
"%	
 7q"=  *+;xPggxx	F ||~4[[]FMr9   c                    V ^8  d   QhR\         R\         R\        R\        R\        \         \         \         3,          /# )rA   r  r  unpack_dataunpack_pivotsrB   r  )rC   s   "r7   rD   rD      sF     $ $$$ $ 	$
 666!"$r9   c                 L  a  \         P                  ! S P                  ^8  V 3R l4       V'       d4   \         P                  ! VP                  \         P                  8H  R 4       \        S P                  4      pVR,          pVR,          p\        WV4      pWTR&   V'       d   S P                  V4      pMS P                  ^ .4      pV'       d0   WtR&   S P                  V4      p	WtR&   WdR&   S P                  V4      p
M$S P                  ^ .4      p	S P                  ^ .4      p
WV
3# )rA   c                  $   < R S P                    R2# )zFtorch.lu_unpack: Expected tensor with 2 or more dimensions. Got size: r  r8  )r  s   r7   rl    lu_unpack_meta.<locals>.<lambda>  s    XY[YaYaXbbjkr9   c                      R # )ztorch.lu_unpack: LU_pivots is expected to be a contiguous tensor of torch.int32 dtype.
Note: this function is intended to be used with the output produced by torch.linalg.lu_factorr3   r3   r9   r7   rl   r    s    pr9   r  r   )	r]   ro   r   re   r  r   r   r  r   )r  r  r  r  r  r  r  r  r  rM  rL  s   f&&&       r7   lu_unpack_metar    s     
LL
1k LLEKK'	
 NEb	Ab	AA	A"ILLLL!b	LLb	b	LLLL!LL!7Nr9   c                R    V ^8  d   QhR\         R\        \        \        3,          /# )rA   moderB   )r  rn   r   )rC   s   "r7   rD   rD   (  s#       tTz!2 r9   c                    a  S R 8X  d   RpRpW3# S R8X  d   RpRpW3# S R8X  d   RpRpW3# \         P                  ! RV 3R l4       XX3# )reducedTcompleteFrc                     < R S  R2# )zqr received unrecognized mode 'z=' but expected one of 'reduced' (default), 'r', or 'complete'r3   )r  s   r7   rl    _parse_qr_mode.<locals>.<lambda>5  s    1$ 8N Or9   r]   ro   )r  	compute_qr  s   f  r7   _parse_qr_moder  (  s    y	  
		  
	  		
 gr9   QRc                ^    V ^8  d   QhR\         R\        R\        \         \         3,          /# )rA   r  r  rB   r   r  rn   )rC   s   "r7   rD   rD   ?  s*      f C ffn8M r9   c                 B   \        V R 4       \        V R 4       \        V4      w  r#V P                  R,          pV P                  R,          p\	        WE4      pV'       dT   \        V P                  4      pV'       d   TMTVR&   V P                  V4      pVP                  V\        VRR7      4       MV P                  ^ .4      p\        V P                  4      p	V'       g	   V'       g   TMTV	R&   V P                  V	4      p
V
P                  V	\        V	RR7      4       W3# )z	linalg.qrFrX  r  r   )	r9  r   r  r   r  r   r   r   r   )r  r  r  reduced_moder  r  r  Q_shaper  R_shaper  s   &&         r7   linalg_qr_metar  =  s     ![!1k*,T2I	A	AA	Aqww-'aQKK 	g:7eTUKK 177mG#9!!GBK	GAMM'6w%PQ4Kr9   sign	logabsdetc                f    V ^8  d   QhR\         R\        \         \         \         \         3,          /# )rA   r  rB   )r   rn   )rC   s   "r7   rD   rD   [  s(     ' 'v '%(F"G 'r9   c                    \        V R 4       \        V R R4       V P                  pV P                  VRR 4      pV P                  VRR \	        V P
                  4      R7      p\        P                  ! V\        VR4      V P
                  V P                  R7      pV P                  VRR \        P                  R7      pW#WE3# )zlinalg.slogdetFNr   r  r  r   )r   r   r   r   rg   re   r]   r  r   r   r  )r  r   r  r  r  r  s   &     r7   _linalg_slogdetr  Y  s     a)*1.6GGE;;uSbz"DE#2Joagg.FGI			*5%8ggxx	
B [[s5;;[7FB&&r9   c          	      V    V ^8  d   QhR\         R\        R\        R\        R,          /# )rA   r  full_matrices
compute_uvdriverNr<  )rC   s   "r7   rD   rD   n  s4     " """ " $J	"r9   c                    \        V R 4       \        V R 4       \        V P                  RR 4      pV P                  R,          pV P                  R,          p\        P
                  ! WV4      pV'       d   YEV'       d   TMT.,           pV P                  V4      p	V	P                  V\        VRR7      4       YA'       d   TMTV.,           p
V P                  V
4      p\        V 4      R8H  pVP                  V
\        WR7      4       M$V P                  ^ .4      p	V P                  ^ .4      pV P                  WG.,           \        V P                  4      R7      pWV3# )z
linalg.svdNFrX  r   r   r  r   )r9  r   r   r   r]   r   r   r   r   r   rg   re   )r  r  r  r  r   r  r  r  U_shaperL  V_shapeVrf  Ss   &&&&          r7   _linalg_svd_metar  m  s    !\"1l+aggcrl#J	A	AaA11==KK 	g:7eTU]1==KK 
 a.F*	g:7VW KKKK 	
J$OAGG,DEA7Nr9   c          	          V ^8  d   QhR\         R\         R\        \        \        ,          \        \        ,          3,          /# )rA   arg1arg2rB   )r   rn   r   r   )rC   s   "r7   rD   rD     s8     . .
.
. 49d3i .r9   c                 &   V P                   R R pVP                   R R p\        W#4      p\        V4      pWPP                  R4      V P                  R4      .,          p\        V4      pWaP                  R4      VP                  R4      .,          pWV3# )Nr  r   )r   r%   r   r   )r  r  arg1_batch_sizesarg2_batch_sizesexpand_batch_portionarg1_expand_sizearg2_expand_sizes   &&     r7   r  r    s    
 zz#2zz#2,-=P012		"66012		"66--r9   c          
      x    V ^8  d   QhR\         R\         R\        R,          R\        \         \         3,          /# )rA   r  r  r   NrB   r  )rC   s   "r7   rD   rD     s=     . .
.
. *. 66>	.r9   c                     V'       d   \        WV4       \        W4      w  r4W0P                  8X  d   T MV P                  V4      pWAP                  8X  d   TMVP                  V4      pWV3# rH   )r*  r  r   rt  )r  r  r   r
  r  arg1_broadcastedarg2_broadcasteds   &&&    r7   rz  rz    sh     t40)Ed)Q& !JJ.DKK@P4Q  !JJ.DKK@P4Q  --r9   c                <    V ^8  d   QhR\         R\         R\        /# )rA   r   r   rB   ro  )rC   s   "r7   rD   rD     s!      f V  r9   c                     V P                   R R pVP                  ^8H  ;'       g8    V P                  ^,
          VP                  8H  ;'       d    VP                   V8H  pV# Nr   )r   r   )r   r   expected_batched_rhs_shapevector_cases   &&  r7   linalg_solve_is_vector_rhsr    s[    !&Sb!1**/  

Q%**$RR8R)R  r9   c                    V ^8  d   QhR\         R\         R\        R\        R\         R,          R\         R,          R\         R,          R	\         R,          R
\        \         \         \         \         3,          /	# )rA   r  r;  r   r  rA  Nr  r  r  rB   r  )rC   s   "r7   rD   rD     s     7 777 	7
 7 TM7 	7 TM7 4-7 6666)*7r9   c                V  a a \        S R 4       \        P                  ! S P                  SP                  8H  V V3R l4       \	        S S4      pV'       d   SP                  R
4      MSp	\        S WR 4       \        V	S 4      w  r\        P                  ! T;'       g    V'       * R 4       V'       d   V
RR
 MT
p\        P                  ! V\        W'       * 4      SP                  SP                  R7      pS P                  p\        P                  ! V\        VR4      S P                  S P                  R7      pS P                  VRR
 \        P                  R7      pS P                  VRR \        P                  R7      pWEWg3pWVV3p\        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       de   \        VV4       FT  w  pp\!        VVP                  4       VP#                  VP                  VP%                  4       4       \'        VVRR	7       KV  	  V# )zlinalg.solvec                  >   < R S P                    RSP                    R2# )zKlinalg.solve: Expected A and B to have the same dtype, but found A of type r  r  r   )r  r;  s   r7   rl   "_linalg_solve_ex.<locals>.<lambda>  s     Ywwiqwwix9r9   c                      R # )zlinalg.solve: Vector broadcasting of the left hand side is not supported for left=False. In this case linalg.solve is equivalent to B / A.squeeze(-1)r3   r3   r9   r7   rl   r    s    Kr9   Nr  Fr   c              3   (   "   T F  qR Jx  K
  	  R # 5irH   r3   )r{   r<   s   & r7   r}   #_linalg_solve_ex.<locals>.<genexpr>  s     
&#QD=#s   T)	copy_fromcopy_toexact_dtyper   r  )r   r]   ro   re   r  	unsqueezer@  r  r  r   r   r   r   r  rZ  r[  r!   r   r   r#   )r  r;  r   r  rA  r  r  r  r  B_B_broad_shaperY   result_shaperesult_r   LU_pivots_info_r   resr  os   ff$$$$$$              r7   _linalg_solve_exr*    s    1n-	LL	177	
 -Q2K'RQBa>23B:M	LLK	
 *5="%-L!!*<Bggxx	G GGE


*5%8ggxx	C kk%*EKKk8GKKcr
%++K6Ev
$C%
(C
s
&#
&sss
&#
&&&SMDAqa)MM!''188:.QuE " Jr9   unitriangularr   c                z    V ^8  d   QhR\         R\         R\        R\        R\        R\         R,          R\         /# )	rA   r  r;  rP  r   r+  r   NrB   ro  )rC   s   "r7   rD   rD     sR       	
   
$ r9   c                    Vf   V P                  ^ .4      p\        V\        4      '       g   \        R\	        V4       24      h\        WVR4       \        WR 4      w  rgVP                  RR4      P                  4       ;'       d    VP                  4       pV'       d   \        WVP                  4      pV# \        WVP                  4      '       d>   VP                  VP                  RR4      P                  4       VP                  RR4       V# )Nzout must be TensorLike, got zlinalg.solve_triangularr  r   )r   ry   r   r   r   r@  rz  rk  r  is_conjr!   r   r"   r   
transpose_)	r  r;  rP  r   r+  r   r!  A_avoid_copy_As	   &&$$$$   r7   linalg_solve_triangular_metar2    s     {kk1#c:&&;DI;GHHaD";<.qT:FB<<B'557HHBJJLLXX. J  XX..KKR,223NN2r"Jr9   XM)r  c                    V ^8  d   QhR\         R\         R\        R\        R\        R\        \         \         3,          /# )rA   r   r  rP  rk  r+  rB   r  )rC   s   "r7   rD   rD     sM     +( +(
+(+( +( 	+(
 +( 66>+(r9   c                   a a \         P                  ! S P                  ^8  V 3R l4       \         P                  ! SP                  ^8  V3R l4       \        S SR4       SP                  \         P
                  8X  d   \        S S4      w  rV\         P                  ! V\        VRR7      S P                  S P                  R7      p\         P                  ! V\        VRR7      SP                  SP                  R7      pWx3# SP                  \         P                  8X  g    SP                  \         P                  8X  d,   \         P                  ! S 4      pS P                  ^ .4      pWx3# \         P                  ! RR 4       XX3# )rA   c                  $   < R S P                    R2# )zMtorch.triangular_solve: Expected b to have at least 2 dimensions, but it has ru  rv  r   s   r7   rl   'triangular_solve_meta.<locals>.<lambda>!  s    ))$79r9   c                  $   < R S P                    R2# )zMtorch.triangular_solve: Expected A to have at least 2 dimensions, but it has ru  rv  r(  s   r7   rl   r8  (  r  r9   triangular_solveFrX  r  c                      R # )z+triangular_solve: Got an unexpected layout.r3   r3   r9   r7   rl   r8  B  s    $Qr9   )r]   ro   r   r*  r   stridedr  r  r   re   r   
sparse_csr
sparse_bsrr   r   )	r   r  rP  rk  r+  self_broadcast_sizeA_broadcast_sizesolutioncloned_coefficients	   ff&&&    r7   triangular_solve_metarC    sC    
LL		Q	
 
LL	!	
 4$67xx5== 0LTST0U-&&$./BeT**;;	
 #00!./?5Q''88	
 '' 
U%%	%U5E5E)E##D)!^^QC0 '' 	UQR'''r9   c                 j   \        V R 4       \        V R 4       V P                  V P                  RR 4      pV P                  V P                  4      pVP	                  V P                  \        V P                  RR7      4       V P                  V P                  RR \        P                  R7      pWV3# )z
linalg.detNFrX  r   r  r   r  )r  detr  r  s   &   r7   _linalg_det_metarF  G  s    a&1l+
++aggcrl
#C	
QWW	BNN17775QR[["U[[[9FF?r9   c                `    V ^8  d   QhR\         R\         R\         R\        R\        R\         /# )rA   r   r  r   r   rk  rB   ro  )rC   s   "r7   rD   rD   W  sL     W WW	W W 	W
 W Wr9   c                   a aaaaa \         P                  ! S P                  ^8  R 4       \         P                  ! SP                  ^8  R 4       V'       d   RMRo\         P                  ! SP                  S,          SP                  R,          8  V3R l4       \         P                  ! SP                  S,          S P                  R,          8H  V3R l4       \         P                  ! SP                  R,          S P                  R,          8*  R 4       \         P                  ! S P                  SP                  ,
          ^8H  V V3R l4       \         P                  ! S P                  SP                  8H  V V3R l4       S P                  ^8  dj   S P                  RR pSP                  RR o\         P                  ! SV8H  V3R	 l4       SP                  RR o\         P                  ! SV8H  V3R
 l4       \         P                  ! SP                  S P                  8H  V V3R l4       \         P                  ! SP                  S P                  8H  V V3R l4       \        RSS R4       \        RSS R4       \         P                  ! SP                  \        SP                  RR7      SP                  SP                  R7      # )rA   c                      R # )z3torch.ormqr: input must have at least 2 dimensions.r3   r3   r9   r7   rl   ormqr.<locals>.<lambda>_      !Vr9   c                      R # )z3torch.ormqr: other must have at least 2 dimensions.r3   r3   r9   r7   rl   rJ  b  rK  r9   c                     < R S  R2# )torch.ormqr: other.shape[z0] must be greater than or equal to tau.shape[-1]r3   left_size_conditions   r7   rl   rJ  h  s    +,?+@@pqr9   c                     < R S  R2# )rN  z"] must be equal to input.shape[-2]r3   rO  s   r7   rl   rJ  l  s    +,?+@@bcr9   c                      R # )zHtorch.ormqr: tau.shape[-1] must be less than or equal to input.shape[-1]r3   r3   r9   r7   rl   rJ  q  r  r9   c                  <   < R SP                    RS P                    2# )z[torch.ormqr: Expected tau to have one dimension less than input, but got tau.ndim equal to r  rv  r  s   r7   rl   rJ  v  r  r9   c                  <   < R SP                    RS P                    2# )zhtorch.ormqr: Expected other to have the same number of dimensions as input, but got other.ndim equal to r  rv  r   r   s   r7   rl   rJ  }  s&    ++0::,6RSXS]S]R^`r9   Nc                     < R S  2# )zWtorch.ormqr: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r3   r  s   r7   rl   rJ    r  r9   c                     < R S  2# )zYtorch.ormqr: Expected batch dimensions of other to be equal to input.shape[:-2], but got r3   )actual_batch_other_shapes   r7   rl   rJ    s    66N5OQr9   c                  <   < R S P                    RSP                    2# )zPtorch.ormqr: Expected input and tau to have the same dtype, but input has dtype z and tau has dtype r   r  s   r7   rl   rJ    s"    ##(;;-/B399+Or9   c                  <   < R S P                    RSP                    2# )zRtorch.ormqr: Expected input and other to have the same dtype, but input has dtype z and other has dtype r   rU  s   r7   rl   rJ    s"    ##(;;-/DU[[MSr9   ztorch.ormqrr  r   FrX  r  r  r   )	r]   ro   r   r   re   rH  r  r   r   )	r   r  r   r   rk  expected_batch_shaperX  r  rP  s	   fff&& @@@r7   ormqrr\  U  s    
LL

aV 
LL

aV !%""	LL'(CIIbM9q 
LL'(EKKO;c
 
LL		"R(Z
 
LL

SXX"	
 
LL

ejj 	
 zzA~${{3B/!$3B"&::	
 $);;s#3 $(<<	
 
LL		U[[ 	
 
LLu{{"	
 M3u5M5%9[[*5;;%Hkk||	 r9   c                  a aa \         P                  ! \        S4      ^S,          8H  VV3R l4       S P                  pVS^,           8H  pTpV'       * pV'       d3   \	        ^V4       F!  pT;'       d    S P                  V4      ^ 8g  pK#  	  M0\	        V4       F!  pT;'       d    S P                  V4      ^ 8g  pK#  	  \         P                  ! T;'       g    TVV 3R l4       R# )rA   c                  4   < R ^S ,           R\        S4       2# )zpadding size is expected to be z, but got: r   )r   paddings   r7   rl   ,_padding_check_valid_input.<locals>.<lambda>  s    1!c'+c'l^Tr9   c                  J   < R S ^,            RS ^,            RSP                    2# )	Expected zD or zcD (batch mode) tensor with possibly 0 batch size and other non-zero dimensions for input, but got: r8  )r   r   s   r7   rl   ra    s-    aycAgY /AAFOr9   N)r]   ro   r   r   r   r   )r   r`  r   	input_dimis_batch_modevalid_batch_modevalid_non_batch_moder   s   ffd     r7   _padding_check_valid_inputrh    s    	LLGCT
 

I#'*M$,,q)$A/FFEJJqMQ4F % y!A#7#N#NEJJqMQ<N  " 
LL000	
r9   c                $  a aaaa	a
 ^ p^o^pS P                   ^8X  d$   S P                  ^ 4      pS^,          oV^,          p\        S V^R7       Vw  o	o
S P                  V4      pS P                  S4      oSS	,           S
,           oV'       d/   \        P                  ! S	S8  ;'       d    S
S8  VV V	V
3R l4       \        P                  ! S^8  VV3R l4       S P                   ^8X  d   S P                  VS34      # S P                  WES34      # )r   r  c                  4   < R S RS RS  RSP                    2# zcArgument #4: Padding size should be less than the corresponding input dimension, but got: padding (r   ) at dimension 
 of input r8  dim_wr   pad_lpad_rs   r7   rl   _pad1d_common.<locals>.<lambda>  -    %%*G2eWOE7*UZU`U`Tacr9   c                     < R S  RS 2# )z
input (W: z%) is too small. Calculated output W: r3   )input_woutput_ws   r7   rl   rr    s    *WI%J8*Ur9   )r   r   rh  r]   ro   r   )r   r`  is_reflection	dim_planenbatchnplanero  ru  rv  rp  rq  s   f&$   @@@@@r7   _pad1d_commonr{    s    IEFzzQA
Q	ug15LE5ZZ	"FjjG&HGO//	
 
LLAU
 zzQ1229::r9   c                     \        WR R7      # Trw  )r{  r   r`  s   &&r7   meta_reflection_pad1dr         t<<r9   c                    a  \         P                  ! S P                  \         P                  8g  V 3R  l4       \	        S VRR7      # )c                  @   < R S P                   P                  4        R2# )z)"replication_pad1d" not implemented for ''re   __str__r   s   r7   rl   (meta_replication_pad1d.<locals>.<lambda>      =ekk>Q>Q>S=TTUXr9   Fr~  )r]   ro   re   r   r{  r  s   f&r7   meta_replication_pad1dr    5     
LLuzz!X u==r9   c                  a aaaaa ^oV'       g%   \         P                  ! \        V4      ^8H  R 4       SP                  ^8X  d
   S^,          oVw  ooSP	                  S4      pVS,           S,           oV'       d/   \         P                  ! SV8  ;'       d    SV8  VVVV3R l4       \         P                  ! SS P	                  S4      8H  VV V3R l4       SP                  SP                  4      # )r2   c                      R # )z padding size is expected to be 2r3   r3   r9   r7   rl   (_pad1d_backward_common.<locals>.<lambda>	  s    0Rr9   c                  4   < R S RS RS  RSP                    2# rk  r8  rn  s   r7   rl   r    rs  r9   c                  2   < R S RSP                  S 4       2# z(grad_output width unexpected. Expected: , Got: r   ro  grad_outputrv  s   r7   rl   r         :8*GKL\L\]bLcKder9   )r]   ro   r   r   r   r   r   )	r  r   r`  rw  ru  ro  rv  rp  rq  s	   ff&$ @@@@r7   _pad1d_backward_commonr    s    ES\Q&(RSzzQ
LE5jjG&HGO//	
 
LLK$$U++e
 ??5;;''r9   
grad_inputc                     \        WVR R7      # r}  r  r  r   r`  s   &&&r7   meta_reflection_pad1d_backwardr  $  s     "+gTRRr9   c                     \        WVR R7      # )Fr~  r  r  s   &&&r7   meta_replication_pad1d_backwardr  *  s     "+gUSSr9   c                  a aaa	a
aaaaaa ^o^o^ p^p\        S V^R7       S P                  pV^8X  d-   S P                  ^ 4      pS^,          oS^,          oV^,          pVw  ooooS P                  V4      pS P                  S4      o	S P                  S4      o
S	S,           S,           oS
S,           S,           oV'       d]   \        P                  ! SS
8  ;'       d    SS
8  VV VV3R l4       \        P                  ! SS	8  ;'       d    SS	8  VV VV3R l4       \        P                  ! S^8  ;'       g    S^8  V	V
VV3R l4       S P                  ^8X  d   S P                  VSS34      # S P                  WFSS34      # )rA   r  c                  4   < R S RS RS  RSP                    2# rk  r8  rn  s   r7   rl   _pad2d_common.<locals>.<lambda>J  rs  r9   c                  4   < R S RS RS  RSP                    2# zcArgument #6: Padding size should be less than the corresponding input dimension, but got: padding (r   rl  rm  r8  dim_hr   pad_bpad_ts   r7   rl   r  Q  rs  r9   c                      < R S  RS RS RS 2# )z
input (H:  W: z%) is too small. Calculated output H: r3   )input_hru  output_hrv  s   r7   rl   r  Y  s%    	gY /$$,:T(=r9   rh  r   r   r]   ro   r   )r   r`  rw  
dim_slicesry  r   rz  r  ro  r  ru  r  rv  r  rp  rq  r  s   f&$    @@@@@@@@@@r7   _pad2d_commonr  0  s\   EEJFug15::DqyA

a
!(E5%ZZ
#FjjGjjG&H&HGO//	
 	GO//	
 
LLA&&Q	
 zzQ(;<<(CDDr9   c                     \        WR R7      # r}  )r  r  s   &&r7   meta_reflection_pad2dr  e  r  r9   c                    a  \         P                  ! S P                  \         P                  8g  V 3R  l4       \	        S VRR7      # )c                  @   < R S P                   P                  4        R2# )z)"replication_pad2d" not implemented for 'r  r  r  s   r7   rl   (meta_replication_pad2d.<locals>.<lambda>p  r  r9   Fr~  )r]   ro   re   r   r  r  s   f&r7   meta_replication_pad2dr  k  r  r9   c                 `    \         P                  ! V4      p\         P                  ! V4      pWV3# rH   r]   r   )grad_wsaved_vsaved_gsaved_normsr   grad_vgrad_gs   &&&&&  r7   meta_weight_norm_backwardr  u  s,     g&Fg&F>r9   c                   a aaaa ^o^o^ pVP                   pVP                  4       ^8X  d   S^,          oS^,          oV^,          pVw  rVrxVS,          p	VS,          p
W,           V,           oW,           V,           o\        P                  ! SS P	                  S4      8H  VV V3R l4       \        P                  ! SS P	                  S4      8H  VV V3R l4       VP                  VP                   4      # )rA   c                  2   < R S RSP                  S 4       2# r  r   r  s   r7   rl   %meta_pad2d_backward.<locals>.<lambda>  r  r9   c                  2   < R S RSP                  S 4       2# z)grad_output height unexpected. Expected: r  r   r  r  r  s   r7   rl   r         ;H:W[M]M]^cMdLefr9   )r   r   r]   ro   r   r   )r  r   r`  rx  rk   rp  rq  r  r  r  ru  r  ro  r  rv  s   f&&        @@@@r7   meta_pad2d_backwardr  ~  s     EEIJxxzQ

Q	!(E%GG&H&H	LLK$$U++e 
LLK$$U++f >>$**%%r9   c          	        a aaa	a
aaaaaaaaaaa ^o	^o^o^ p\        S V^R7       S P                  ^8H  pV'       d6   S P                  ^ 4      pS	^,          o	S^,          oS^,          oV^,          pVw  ooooooS P                  V4      pS P                  S4      o
S P                  S4      oS P                  S	4      oS
S,           S,           oSS,           S,           oSS,           S,           oV'       d   \        P                  ! SS8  ;'       d    SS8  V	V VV3R l4       \        P                  ! SS8  ;'       d    SS8  VV VV3R l4       \        P                  ! SS
8  ;'       d    SS
8  VV VV3R l4       \        P                  ! S^8  ;'       g    S^8  ;'       g    S^8  V
VVVVV3R l4       V'       d   S P                  XVSSS34      # S P                  VSSS34      # )   r  c                  4   < R S RS RS  RSP                    2# rk  r8  rn  s   r7   rl   _pad3d_common.<locals>.<lambda>  rs  r9   c                  4   < R S RS RS  RSP                    2# r  r8  r  s   r7   rl   r    rs  r9   c                  4   < R S RS RS  RSP                    2# )zcArgument #8: Padding size should be less than the corresponding input dimension, but got: padding (r   rl  rm  r8  )dim_dr   pad_bkpad_fs   r7   rl   r    s-    %%*G2fX_UG:V[VaVaUbdr9   c                  ,   < R S  RS RS RS RS RS 2# )z
input (D:  H: r  z%) is too small. Calculated output D: r3   )input_dr  ru  output_dr  rv  s   r7   rl   r    s2    	gYd7) <$$,:T(4zKr9   r  )r   r`  rw  rx  
batch_modery  rz  r  r  ro  r  r  ru  r  r  rv  r  r  r  rp  rq  r  s   f&$    @@@@@@@@@@@@@@@r7   _pad3d_commonr    s   EEEIug15qJA


Q	07-E5%vZZ	"FjjGjjGjjG'H&H&HGO//	
 	GO//	
 	GO00 0	
 
LLA77Q77(a-	
 	
 (HMNN(HEFFr9   c                     \        WR R7      # r}  )r  r  s   &&r7   meta_reflection_pad3dr    r  r9   c                    a  \         P                  ! S P                  \         P                  8g  V 3R  l4       \	        S VRR7      # )c                  @   < R S P                   P                  4        R2# )z)"replication_pad3d" not implemented for 'r  r  r  s   r7   rl   (meta_replication_pad3d.<locals>.<lambda>  r  r9   Fr~  )r]   ro   re   r   r  r  s   f&r7   meta_replication_pad3dr    r  r9   c                   a aaaaaa \         P                  ! \        V4      ^8H  R 4       VP                  ^8:  d   \	        RVP                   24      hS P                  VP                  8w  d&   \	        RS P                   RVP                   24      h^o^o^oVP                  ^8X  d   S^,          oS^,          oS^,          oVw  r4rVrxVP                  S4      p	VP                  S4      p
VP                  S4      pW,           V,           oW,           V,           oW,           V,           o\         P                  ! SS P                  S4      8H  VV V3R l4       \         P                  ! SS P                  S4      8H  VV V3R l4       \         P                  ! SS P                  S4      8H  VV V3R l4       VP                  VP                  4      # )   c                      R # )z padding size is expected to be 6r3   r3   r9   r7   rl   %meta_pad3d_backward.<locals>.<lambda>  s    ,Nr9   zinput.ndim must be > 3, got z,grad_output.ndim must equal input.ndim, got  != c                  2   < R S RSP                  S 4       2# r  r   r  s   r7   rl   r  	  r  r9   c                  2   < R S RSP                  S 4       2# r  r   r  s   r7   rl   r  	  r  r9   c                  2   < R S RSP                  S 4       2# )z(grad_output depth unexpected. Expected: r  r   )r  r  r  s   r7   rl   r  !	  r  r9   )r]   ro   r   r   r   r   r   r   )r  r   r`  rp  rq  r  r  r  r  r  r  ru  r  r  ro  r  r  rv  s   f&&         @@@@@@r7   meta_pad3d_backwardr    s    
LLW"$NOzzQ;EJJ<HII5::%:;;K;K:LDQVQ[Q[P\]
 	
 EEEzzQ


07-E%jjGjjGjjG'H&H&H	LLK$$U++e 
LLK$$U++f 
LLK$$U++e
 ??5;;''r9   c                <    V ^8  d   QhR\         R\        R\         /# )rA   r   prB   r   ra   )rC   s   "r7   rD   rD   )	  s!     

 

f 

 

v 

r9   c                 f   \         P                  ! V P                  4       R  4       V P                  ^ 4      pV^8:  d1   V P	                  ^ .4      P                  \         P                  R7      # V P	                  W"^,
          ,          ^,          34      P                  \         P                  R7      # )c                      R # )z(_pdist_forward requires contiguous inputr3   r3   r9   r7   rl   %meta__pdist_forward.<locals>.<lambda>+	  s    &Pr9   r   )r]   ro   r  r   r   rr  r  )r   r  r  s   && r7   meta__pdist_forwardr  '	  s     
LLP 			!AAv~~qc"%%E4R4R%SS~~qE{a/125588 6 
 	
r9   c          
      T    V ^8  d   QhR\         R\         R\        R\         R\         /# )rA   gradr   r  pdistrB   r  )rC   s   "r7   rD   rD   8	  s7     P Pv PV P Pf PQW Pr9   c                     \         P                  ! VP                  4       R  4       \         P                  ! VP                  4       R 4       \         P                  ! V\         P                  R7      # )c                      R # )z._pdist_backward requires self to be contiguousr3   r3   r9   r7   rl   &meta__pdist_backward.<locals>.<lambda>:	  s    &Vr9   c                      R # )z/_pdist_backward requires pdist to be contiguousr3   r3   r9   r7   rl   r  =	  s    'Xr9   r   )r]   ro   r  r   r  )r  r   r  r  s   &&&&r7   meta__pdist_backwardr  6	  sW     
LLV 
LLX D0N0NOOr9   c          
        a aaaaa ^ RI HpHp SP                  ^ 4      pSP                  ^4      pSP                  ^4      p	V! \        P
                  ! V! S P                  WxV	34      4      4      '       d   S P                  WxV	34      o \        P                  ! SP                  4       ^8H  R 4       \        P                  ! SP                  4       ^8H  R 4       \        P                  '       gP   \        P                  ! S P                  SP                  u;8H  ;'       d    SP                  8H  Mu VVV 3R l4       SP                  p
SP                  oV
^ ,          oV
^,          o\        P                  ! S^ ,          S8H  ;'       d    S^,          S8H  VVV3R l4       S P                  S P                  4       4      # )r   )guard_or_truesym_eqc                      R # zbatch1 must be a 3D tensorr3   r3   r9   r7   rl   meta_baddbmm.<locals>.<lambda>L	      ,Hr9   c                      R # zbatch2 must be a 3D tensorr3   r3   r9   r7   rl   r  M	  r  r9   c                  V   < R SP                    RS P                    RSP                    2# )z+Input dtypes must be the same, got: input: z
, batch1: z
, batch2: r   )batch1batch2r   s   r7   rl   r  Q	  s.    A$**ZX^XdXdWeeopvp|p|o}~r9   c            	      >   < R S RS RS ^ ,           RS ^,           R2	# z@Expected size for first two dimensions of batch2 tensor to be: [r   z] but got: [].r3   batch2_sizesbscontraction_sizes   r7   rl   r  Y	  s5    t2&'|LO3DB|TUFWWY[r9   )ro  r  r  r   r]   sym_notr   rt  ro   r   
exp_config&skip_dtype_check_in_meta_registrationsre   r   )r   r  r  r  r  r  r  dim1dim2dim3batch1_sizesr  r  r  s   fff$$      @@@r7   meta_baddbmmr  B	  s?    L;;q>D;;q>D;;q>DU]]6$**t46H#IJKK{{D-.	LL"$HI	LL"$HI<<<JJ&,,66&,,6~	
 <<L<<L	aB#A	LLQ2EE,q/5E"E	
 >>$))+&&r9   c                N    \         P                  ! V \         P                  R 7      # r   r  r   r  s   &$r7   meta_bernoullir  a	  s     D0G0GHHr9   c                     V # rH   r3   r   r  r  s   &&&r7   meta_bernoulli_r  h	      Kr9   c                 N    \         P                  ! V \         P                  R 7      # r   r  r  s   &&&r7   meta_bernoulli_pr  m	  s     D0G0GHHr9   c                 .    \         P                  ! V 4      # rH   r  r  s   &&r7   meta_poissonr  s	       D!!r9   c                     \         P                  ! WP                  4       8  R  4       \         P                  ! V \         P                  R7      p\         P                  ! V 4      V3# )c                      R # )zJError in fused_moving_avg_obs_fake_quant_cpu: ch_axis must be < self.dim()r3   r3   r9   r7   rl   6meta__fused_moving_avg_obs_fq_helper.<locals>.<lambda>	      \r9   r   )r]   ro   r   r   r   )r   observer_onfake_quant_onrunning_minrunning_maxscale
zero_pointaveraging_const	quant_min	quant_maxch_axisper_row_fake_quantsymmetric_quantmasks   &&&&&&&&&&&&& r7   $meta__fused_moving_avg_obs_fq_helperr*  y	  sM      
LL((*\ D

3DT"D))r9   c                F    V ^8  d   QhR\         P                  R,          /# )rA   r  N)r]   re   )rC   s   "r7   rD   rD   	  s     3 3U[[4/ 3r9   c                 j  aaaa \         P                  ! V P                  4       ^8H  R 4       \         P                  ! VP                  4       ^8H  R 4       V P                  w  ooVP                  w  oo\         P                  ! SS8H  VVVV3R l4       Veu   \         P                  ! W P                  8H  ;'       gI    V\         P
                  8H  ;'       d.    V P                  \         P                  \         P                  39   R 4       Vf   V P                  MTpV P                  SS3VR7      # )rA   c                      R # )za must be 2Dr3   r3   r9   r7   rl   meta_mm.<locals>.<lambda>	      ~r9   c                      R # )zb must be 2Dr3   r3   r9   r7   rl   r.  	  r/  r9   c            	      "   < R S RS  RS RS R2	# )z/a and b must have same reduction dim, but got [r   z] X [r   r3   )M1M2Nr  s   r7   rl   r.  	  s&    A!Brd%PRtSUVWUXXZ[r9   c                      R # )zFout_dtype must be the same as input dtype or fp32 for fp16/bf16 inputsr3   r3   r9   r7   rl   r.  	  s    \r9   r   )	r]   ro   r   r   re   r  r  r  r   )r5   r6   r  rZ   r2  r3  r4  r  s   &&& @@@@r7   meta_mmr6  	  s     
LLA56	LLA56GGEArGGEB	LL
b[    U]]* ? ?GGu~~>>\	
 (/177YL;;1v\;22r9   c                   a a V'       d[   \         ;QJ d+    . VV 3R  l\        S P                  4       4       F  NK  	  5# ! VV 3R  l\        S P                  4       4       4      # \        P                  ! S P
                  S4      # )c              3   Z   <"   T F   qS9  d   SP                   V,          M^x  K"  	  R# 5irS  r8  )r{   r   dimsr   s   & r7   r}   +_compute_reduction_shape.<locals>.<genexpr>	  s$     UDTqtmTZZ]:DTs   (+)rn   r   r   rV   compute_reduction_output_shaper   )r   r9  r  s   ff&r7   r  r  	  sQ    uUE$))DTUuUuUE$))DTUUU//

DAAr9   c                    V ^8  d   QhRR/# )rA   rB   r  r3   )rC   s   "r7   rD   rD   	  s     
 
5 
r9   c                 D   \        V \        P                  P                  4      '       d   V P                  P
                  # \        V R 4      '       dN   \        V P                  R4      '       d2   V P                  P
                  R8w  d   V P                  P
                  # R# )r   r   r   r   )ry   r]   _subclasses
FakeTensorfake_devicer   hasattrr   )r{  s   &r7   r   r   	  sp    &%++6677!!&&&!!FMM6**MM&(}}!!!r9   c                ^   V ^8  d   QhR\         P                  R\         P                  R\        \        ,          \        ,          R\        \        ,          \        ,          R\        \        ,          \        ,          R\        R\        R\        \        ,          \        ,          R	,          /# )
rA   input_tensorr  r   r`  dilationis_transposedgroupsoutput_paddingNr]   r   r   r   r   )rC   s   "r7   rD   rD   	  s     D D,,DLLD IOD #Y_	D
 3i#oD D D IOd*Dr9   c                   aa R  R lpR R lp	VP                   R,          p
V P                   R,          oV'       d   WaP                   ^,          ,          pMWVP                   ^ ,          p\        P                  ! VP                   ^,          V,          V P                   ^,          8H  R 4       V P                   ^ ,          V.o\        V\        4      '       d   V.\        S4      ,          pM*\        V4      ^8X  d   V^ ,          .\        S4      ,          p\        V\        4      '       d   V.\        S4      ,          pM*\        V4      ^8X  d   V^ ,          .\        S4      ,          p\        V\        4      '       d   V.\        S4      ,          pM*\        V4      ^8X  d   V^ ,          .\        S4      ,          pRpV'       dX   \        V\        4      '       d   V.\        S4      ,          pM-\        V4      ^8X  d   V^ ,          .\        S4      ,          pMTp\        \        S4      4       F  pV'       dD   SP                  V	! SV,          W=,          WM,          W,          W-,          W,          4      4       KN  SP                  V! SV,          W=,          WM,          W,          W-,          4      4       K  	  ^ RIH	p ^ RI
Hp \        W4      '       d   V P                  MV P                  pVP                  R	8H  ;'       d    \        P                  P                   RJ pV'       g;   \        P                  ! T! SR,           Uu. uF  pV^ 8  NK
  	  up!  VV3R
 l4       S# u upi )c                `    V ^8  d   QhR\         R\         R\         R\         R\         R\         /# )rA   lnr  r   r  r  rB   r   )rC   s   "r7   rD   /calc_conv_nd_return_shape.<locals>.__annotate__	  s6     7 7S 7S 7S 7S 7S 7S 7r9   c                t    V ^V,          ,           W#^,
          ,          ,
          ^,
          V,          ^,           # )aE  
Formula to apply to calculate the length of some dimension of the output

See: https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

Args:
    ln: length of the dimension
    p: padding in that dim
    d: dilation in that dim
    k: kernel size in that dim
    s: stride in that dim
Returns:
    The output length
r3   )rK  r  r   r  r  s   &&&&&r7   _formula+calc_conv_nd_return_shape.<locals>._formula	  s,     QU
Qa%[(1,2Q66r9   c                l    V ^8  d   QhR\         R\         R\         R\         R\         R\         R\         /# )rA   rK  r  r   r  r  rI   rB   rL  )rC   s   "r7   rD   rM  	  s>     ; ; ; ; ; ; ; ;QT ;r9   c                    V ^,
          V,          ^V,          ,
          W#^,
          ,          ,           V,           ^,           # )a  
Formula to apply to calculate the length of some dimension of the output
if transposed convolution is used.
See: https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html

Args:
    ln: length of the dimension
    p: padding in that dim
    d: dilation in that dim
    k: kernel size in that dim
    s: stride in that dim
    op: output padding in that dim

Returns:
    The output length
r3   )rK  r  r   r  r  rI   s   &&&&&&r7   _formula_transposed6calc_conv_nd_return_shape.<locals>._formula_transposed	  s0    " Q!|a!e#aq5k1B6::r9   rA   NNc                      R # )zInvalid channel dimensionsr3   r3   r9   r7   rl   +calc_conv_nd_return_shape.<locals>.<lambda>	  s    0r9   N)r?  )sym_orr   c                  6   < R \        S 4       RSR,           R2# )zGiven input size per channel: z&. Calculated output size per channel: rU  z. Output size is too small)r   )r9  	ret_shapes   r7   rl   rW  @
  s(    4T$ZL A33<R=/ B'(r9   )r   r]   ro   ry   r   r   r   appendtorch._subclasses.fake_tensorr?  ro  rX  r@  r   r   versionhip)rC  r  r   r`  rD  rE  rF  rG  rO  rS  kernel_sizeout_channelsoutput_padding_listr   r?  rX  r   is_cudnnr<   r9  rZ  s   &&&&&&&&           @@r7   calc_conv_nd_return_shaperc  	  s   7";& ,,r"Kb!DQ/||ALLOf$(:(:1(==0	

 ##A&5I&'""CI%	V	)s4y('7##)c$i'	W	1:,T*(G$$:D	)	X!	QK=3t9,,0ng..#1"2SY"> A%#1!#4"5D	"A"03t9#GJ KNI'* a'*hk;>69U% 6 9< l// 	      {{f$BB):):d)BHIbM2MqQUM23(	
  3s   M,c                 b    \         P                  P                  V 4      \         P                  8H  # rH   r]   _prims_commonr   channels_lasttens   &r7   is_channels_lastrj  H
  s$    44S9U=P=PPPr9   c                   V ^8  d   QhR\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R\        R\        R	\        /# )
rA   rC  r  r  Nrunning_meanrunning_vartrainingexponential_average_factorepsilon)r]   r   r   ra   )rC   s   "r7   rD   rD   M
  s{     "$ "$,,"$LL"$ ,,
"$ ,,%	"$
 $"$ "$ !&"$ "$r9   c                   a  S P                   pVe   VP                   MVP                   p	Ve   VP                   MVP                   p
V 3R lpS P                  V4      P                  V! 4       R7      pV'       d$   S P                  V	4      pS P                  V
4      pM"S P                  R4      pS P                  R4      pWV3# )Nc                     < \        S 4      '       d   \        P                  # S P                  \        P                  R 7      '       d   \        P                  # \        P                  # r   )rj  r]   rg  r  r   )rC  s   r7   pick_memory_format2meta_miopen_batch_norm.<locals>.pick_memory_format_
  sL    L))&&&%%E4K4K%LL***&&&r9   r   r  )r   r   rr  )rC  r  r  rl  rm  rn  ro  rp  r   save_mean_shapesave_var_shapers  r   	save_meansave_vars   f&&&&&&&       r7   meta_miopen_batch_normry  L
  s     ""I -9,Dl((&,,O*5*A[&&v||N' 
 
 
+
.
.=O=Q
.
RC **?;	)).9 **40	))$/8##r9   c                   V ^8  d   QhR\         P                  R\         P                  R\         P                  R\        \        ,          R\        \        ,          R\        \        ,          R\        R\        \        ,          R	\        /	# )
rA   rC  r  r  r   r`  rD  rE  rG  rF  rH  )rC   s   "r7   rD   rD   s
  s~     # #,,#LL# ,,# I	#
 #Y# 3i# # I# #r9   c	           
          \        T TTTTTTV'       d   TMR 4      p	^ RIHp
 ^p^pV
! V P                  V4      ^ 8H  4      '       d   ^ W&   V P	                  V	4      pV# )Nguard_or_false)rc  ro  r}  r   r   )rC  r  r  r   r`  rD  rE  rG  rF  	shape_outr}  input_channels_dimoutput_channels_dimr   s   &&&&&&&&&     r7   	meta_convr  r
  sw     *'T	I El''(:;q@AA)*	& 
 
 
+CJr9   mkldnnc
           
          \        WWCVR V. 4      p
V P                  V
4      p\        P                  pV P	                  4       ^8X  d   \        P
                  pVP                  VR7      pV# )Fr   )rc  r   r]   rg  r   channels_last_3drr  )rC  r  r  r`  r   rD  rF  attrscalars	algorithmr~  r   out_memory_formats   &&&&&&&&&&   r7   meta_mkldnn_convolution_defaultr  
  sl     .&8UFB
	 $$Y/!//" % 6 6ff#4f5
r9   c                 j    V P                  . V P                  R R OVP                  ^ ,          N54      # r  r   r   )rC  r  r  r  r  r  s   &&&&&&r7   meta_linear_pointwise_defaultr  
  s5     %%&Q(:(:3B(?&Qa&QRRr9   r  c                 j    V P                  . V P                  R R OVP                  ^ ,          N54      # r  r  )rC  packed_weightorig_weightr  r   s   &&&&&r7   meta_mkl_linearr  
  s:    ))@,$$Sb)@;+<+<Q+?@ r9   onednnc           
      (   \        V VVVV	R V
R4      pVf   V P                  pV\        P                  \        P                  \        P
                  \        P                  \        P                  39  d   \        RV 24      hV P                  VVR7      p\        V4      R9  d   \        R\        V4       R24      h^\        P                  ^\        P                  ^\        P                  /\        V4      ,          pVP                  VR7      pV# )FNOoutput_dtype must be one of float32, bfloat16, uint8, int8, float8_e4m3fn, got r   z3Expect output to be 3d/4d/5d for conv1d/2d/3d, got r   r   )r        )rc  re   r]   r  r  uint8r  r  r   r   r   r   rg  r  rr  )r<   x_scalex_zpww_scalew_zpr  r   r`  rD  rF  output_scaleoutput_zero_pointoutput_dtyper  r  r  r~  r   rC   s   &&&&&&&&&&&&&&&&&   r7   meta_qconv_pointwiser  
  s   , .	
	 77LMMNNKKJJ 
 
 !abnaop  kk)<k8y>* Ec)nEUUVW  u&&u""u%%
 i.	
 ff6f*
r9   c                 2    VR 8w  d   \        RV R24      hV# )sumz#binary_op_name must be 'sum', got 'r  r   )r<   r  r  r  r  r  accumr  r   r`  rD  rF  r  r  r  accum_scaleaccum_zero_pointbinary_op_namer  unary_op_nameunary_op_argsunary_op_algorithms   &&&&&&&&&&&&&&&&&&&&&&r7   meta_qconv2d_pointwise_binaryr    s-    4 U" 5n5EQG  r9   c                 @   \        V P                  4      pVP                  ^,          VR&   V	\        P                  \        P                  \        P
                  \        P                  \        P                  39  d   \        RV	 24      hV P                  WR7      pV# )r2   zOoutput_dtype must be one of float32, bfloat16, int8, uint8, float8_e4m3fn, got r   r   )
r   r   r]   r  r  r  r  r  r   r   )r<   r  r  r  r  r  r  r  r  r  post_op_namepost_op_argspost_op_algorithmr  r   s   &&&&&&&&&&&&&  r7   meta_qlinear_pointwiser  #  s    " AGG}771:RMMNNJJKK 
 
 !abnaop  kk,k;
r9   c                 T   VR 8X  d   V# \        V P                  4      pVP                  ^,          VR&   V
\        P                  \        P                  \        P
                  \        P                  \        P                  39  d   \        RV
 24      hV P                  VV
R7      pV# )r  r  r   r   )
r   r   r]   r  r  r  r  r  r   r   )r<   r  r  r  r  r  x_2r  r  r  r  x2_scalex2_zpr  r  r  r  r  r  r   s   &&&&&&&&&&&&&&&&&&  r7   meta_qlinear_pointwise_binaryr  D  s    , U"JAGG}771:RMMNNKKJJ 
 
 !abnaop  kk,lk;
r9   c                 ~    \        V P                  4      pVP                  ^,          VR&   V P                  V4      pV# r2   r   )r   r   r   )r<   r  r  r  r   s   &&&  r7   meta_linear_dynamic_fp16r  l  s6     AGG}771:Rkk,'
r9   	quantizedc                 ,   \        WW#WE4      w  pppV P                  4       ^8X  d   V P                  R4      M^p	\        P                  p
V P                  4       ^8X  d   WgV.pMWWx.p\        P
                  ! VV P                  V P                  V
R7      # r  r  )#max_pool2d_checks_and_compute_shaper   r   r]   rg  r   re   r   r   r_  r   r`  rD  	ceil_modenInputPlaneoutputHeightoutputWidthry  r   r   s   &&&&&&      r7   meta_quantized_max_pool2dr  }  s     0
		
 $)99;!#3B++99;!{;DCD{{++<<'	
 	
r9   c                   a aaa \         P                  ! S P                  4       ^8H  V 3R l4       \         P                  ! SP                  4       ^8H  V3R l4       \         P                  ! S P                  \         P                  \         P
                  \         P                  39   V 3R l4       \         P                  ! SP                  \         P                  8H  V3R l4       \         P                  ! SP                  \         P                  8H  V3R l4       \         P                  ! SP                  S P                  8H  V3R l4       S P                  S P                  ^ 4      SP                  ^ 4      S P                  R7      # )rA   c                  ,   < R S P                  4        R2# )zx must be a 2D tensor, got r  r  r<   s   r7   rl   /meta_int4mm_packed_weight_cpu.<locals>.<lambda>      -H	QR+Sr9   c                  ,   < R S P                  4        R2# )zw must be a 2D tensor, got r  r  r  s   r7   rl   r    r  r9   c                  "   < R S P                    2# z#expected x to be f32/f16/bf16, got r   r  s   r7   rl   r    s    9!''Cr9   c                  "   < R S P                    2# zexpected w to be uint8, got r   r  s   r7   rl   r    s    .J177),Tr9   c                  "   < R S P                    2# )z q_group_size must be int64, got r   )q_group_sizes   r7   rl   r    s    6|7I7I6JKr9   c                  "   < R S P                    2# )z5q_scale_and_zeros must have the same dtype as x, got r   )q_scale_and_zeross   r7   rl   r    s    KL]LcLcKder9   r   )r]   ro   r   re   r  r  r  r  r   r   r   r<   r  r  r  s   ffffr7   meta_int4mm_packed_weight_cpur    s    QUUW\#STQUUW\#STGGu}}ennEEC	
 	GGu{{"$T	
 	%++-K	
 	##qww.e	
 {{166!9affQiqww{??r9   c                    a aaa \         P                  ! S P                  4       S8H  ;'       d    S P                  S,          S8H  VVVV 3R  l4       R# )c                  z   < R S  RS RS R2RSP                  4        RS RSP                  S,           2,           # )zExpected a tensor of dimension z and tensor.size[z] == r   zbut got : dimension z] = r   r   )r   dim_sizer   r{  s   r7   rl    check_dim_size.<locals>.<lambda>  sN    1#6GzQVW[V\\^_ .?zfll[cNdMe
fgr9   N)r]   ro   r   r   )r{  r   r  r   s   ffffr7   check_dim_sizer    s9    	LL

>>X 6$ >	gr9   c          
          V ^8  d   QhR\         P                  R\        R\        R\         P                  R\         P                  /# )rA   r   r!  r"  re   rB   )r]   r   ra   r   re   )rC   s   "r7   rD   rD     s@     # #<<# %#36#?D{{#
\\#r9   c                 .    \         P                  ! V 4      # rH   r  )r   r!  r"  re   s   &&&&r7   meta_quantize_per_tensorr    s     E""r9   c                    a  R  pV! RV4      w  r\         P                  ! \        V4      R	9   R 4       \         P                  ! S P                  \         P                  \         P
                  \         P                  \         P                  39  V 3R l4       \        V4      ^ 8X  d   YrM-\        V4      ^8X  d   V^ ,          V^ ,          rMV! RV4      w  rV! RV4      w  r\         P                  ! VRJ ;'       g    V^ 8g  R 4       S P                  4       ^8X  d   S P                  R
4      M^pS P                  R4      pS P                  R4      pS P                  R4      p\        VWV
^V4      p\        VWV^V4      p\        P                  ! S 4      p\        S VV	V
VVV^^VVVVVV4       S P                  4       ^8X  d   VVV.pMWVV.p\         P                  ! VS P                  S P                  VR7      # )c                    a  \         P                  ! \        V4      R9   V 3R l4       V^ ,          p\        V4      ^8X  d   TMV^,          pW#3# )r2   c                     < R S  R2# )zavg_pool2d: 4 must either be a single int, or a tuple of two intsr3   r   s   r7   rl   1meta_avg_pool2d.<locals>.unpack.<locals>.<lambda>      l4&(\]r9   r2   rA   r]   ro   r   r   r  HWs   f&  r7   unpackmeta_avg_pool2d.<locals>.unpack  E    H]	
 FSQACFtr9   r_  c                      R # zOavg_pool2d: stride must either be omitted, a single int, or a tuple of two intsr3   r3   r9   r7   rl   !meta_avg_pool2d.<locals>.<lambda>      ar9   c                  @   < R S P                   P                  4        R2# )z""avg_pool2d" not implemented for 'r  r  r  s   r7   rl   r        6u{{7J7J7L6MQQr9   r   r`  Nc                      R # zdivisor must be not zeror3   r3   r9   r7   rl   r        *r9   r  r   r2   rA   r  r  r   )r]   ro   r   re   r  uint16uint32r>  r   r   pooling_output_shaperV   r   pool2d_shape_checkr   r   )r   r_  r   r`  r  count_include_paddivisor_overrider  kHkWdHdWpadHpadWry  r  inputHeight
inputWidthr  r  r   r   s   f&&&&&&               r7   meta_avg_pool2dr    s    M;/FB	LLFy a 
LLEKKu||U\\RRQ 6{aB	V	F1IB&)	7+JD	LLD 99$4$9*
  %yy{a/UZZ^QF**R.K**R.KBJ'Rr1iPL&z2RINK//6M



		$ yy{a\;7\;?;;kk||#	 r9   c                     \        V VVVVVV^^V	V
VVVV4       V P                  4       pT	p\        WV^,
          V4       \        WV^,
          V4       \        WV^,
          V4       R# rS  )r  r   r  )r   
gradOutputry  r  r  r  r   r  r  r  r  r  r  r  
mem_formatr   nOutputPlanes   &&&&&&&&&&&&&&&  r7   avg_pool2d_backward_shape_checkr
    s{    " 



		$ 99;DL:TAX|<:TAX|<:TAX{;r9   c                 |   \         P                  ! \        V4      ^8H  ;'       g    \        V4      ^8H  R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p	\         P                  ! \        V4      ^ 8H  ;'       g%    \        V4      ^8H  ;'       g    \        V4      ^8H  R 4       \        V4      ^ 8X  d   TMV^ ,          p
\        V4      ^ 8X  d   T	M\        V4      ^8X  d   T
MV^,          p\         P                  ! \        V4      ^8H  ;'       g    \        V4      ^8H  R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\         P                  ! VRJ ;'       g    V^ 8g  R 4       VP                  pVP	                  4       ^8X  d
   VR,          M^pVR,          pVR	,          pVR
,          p\        VWV
^V4      p\        VWV^V4      p\        P                  ! V4      p\        VV VVV	V
VVVVVVVVV4       \         P                  ! VVP                  VP                  VR7      # )r2   c                      R # )zKavg_pool2d: kernel_size must either be a single int, or a tuple of two intsr3   r3   r9   r7   rl   *meta_avg_pool2d_backward.<locals>.<lambda>L  s    ]r9   c                      R # r  r3   r3   r9   r7   rl   r  R  r  r9   c                      R # )zGavg_pool2d: padding must either be a single int, or a tuple of two intsr3   r3   r9   r7   rl   r  X  s    Yr9   Nc                      R # r  r3   r3   r9   r7   rl   r  _  r  r9   r  r  r  r  r   )r]   ro   r   r   r   r  rV   r   r
  r   re   r   )gradOutput_r   r_  r   r`  r  r  r  r  r  r  r   r  r  
input_sizery  r  r  r  r  r  r  s   &&&&&&&&              r7   meta_avg_pool2d_backwardr  >  s    
LLKA66[!1Q!6] 
QB;1$+a.B	LLFq@@CK1,@@Fq0@a 6{aVAYB6{a3v;!+;RB	LLG..S\Q.Y 1:Dw<1$4'!*D	LLD 99$4$9*
 J$yy{a/Z^QFR.KR.KBJ'Rr1iPL&z2RINK,,U3J#



$ ;;kk|| 	 r9   c                   a  \         P                  ! \        V4      R
9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p	\         P                  ! V'       * ;'       g    \        V4      R
9   R 4       \         P                  ! S P                  \         P                  \         P
                  \         P                  \         P                  39  V 3R l4       V'       g   TMV^ ,          p
V'       g   TM\        V4      ^8X  d   T
MV^,          pV'       g   T	M\        V4      ^8X  d   T
MV^,          p\         P                  ! \        V4      R
9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p\         P                  ! S P                  R9   R 4       \         P                  ! V'       * ;'       g    V^ 8g  R 4       S P                  ^ 4      pS P                  R4      pS P                  R4      pS P                  R4      pS P                  R4      p\        VW}V
^V4      p\        VWV^V4      p\        VWV^V4      p\        S VVVV	V
VVVVV^^^VVVVVVRRR	7       S P                  ^8X  d   S P                  VVVV34      # S P                  VVVVV34      # )r2   c                      R # zFavg_pool3d: kernel_size must be a single int, or a tuple of three intsr3   r3   r9   r7   rl   !meta_avg_pool3d.<locals>.<lambda>      Xr9   c                      R # zJavg_pool3d: stride must be omitted, a single int, or a tuple of three intsr3   r3   r9   r7   rl   r    r  r9   c                  @   < R S P                   P                  4        R2# )z""avg_pool3d" not implemented for 'r  r  r  s   r7   rl   r    r  r9   c                      R # zBavg_pool3d: padding must be a single int, or a tuple of three intsr3   r3   r9   r7   rl   r        Tr9   c                      R # z9non-empty 4D or 5D (batch mode) tensor expected for inputr3   r3   r9   r7   rl   r        Kr9   c                      R # r  r3   r3   r9   r7   rl   r    r  r9   zavg_pool3d()T)check_input_sizer2   r  r  r  r  r  r  r   )r]   ro   r   re   r  r  r  r>  r   r   r  pool3d_shape_checkr   )r   r_  r   r`  r  r  r  kTr  r  dTr  r   padTr  r  ry  nslicesitimeiheightiwidthotimeoheightowidths   f&&&&&&                 r7   meta_avg_pool3dr1    s    
LLKF"X 
QB;1$+a.B;1$+a.B	LL
++c&kV+\ 
LLEKKu||U\\RRQ vayBc&kQ&6F1IBc&kQ&6F1IB	LLGT 1:Dw<1$4'!*Dw<1$4'!*D	LL

fK
 
LL55 0A 5*
 ZZ]FjjnGJJrNEjjnGZZ^F "aCE"7Bb!YGG!&"B9EF





			-2 zzQ@AAHIIr9   c                    \         P                  ! \        V4      R9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p	\        V4      ^8X  d   TMV^,          p
\         P                  ! V'       * ;'       g    \        V4      R9   R 4       V'       g   TMV^ ,          pV'       g   T	M\        V4      ^8X  d   TMV^,          pV'       g   T
M\        V4      ^8X  d   TMV^,          p\         P                  ! \        V4      R9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p\         P                  ! VP                  R9   R 4       \         P                  ! V'       * ;'       g    V^ 8g  R 4       VP	                  R	4      pVP	                  R
4      pVP	                  R4      pVP	                  R4      p\        VWV^V4      p\        VWV^V4      p\        VV
VV^V4      p\        VV VVV	V
VVVVVVVVVVVVR4       VP                  VP                  4      # )r2   c                      R # r  r3   r3   r9   r7   rl   *meta_avg_pool3d_backward.<locals>.<lambda>  r  r9   c                      R # r  r3   r3   r9   r7   rl   r4    r  r9   c                      R # r  r3   r3   r9   r7   rl   r4    r  r9   c                      R # r   r3   r3   r9   r7   rl   r4    r!  r9   c                      R # r  r3   r3   r9   r7   rl   r4    r  r9   zavg_pool3d_backward()r$  r%  r  r  r  r   )	r]   ro   r   r   r   r  avg_pool3d_backward_shape_checkr   r   )r  r   r_  r   r`  r  r  r  r'  r  r  r(  r  r   r)  r  r  r*  r+  r,  r-  otime_for_shape_checkoheight_for_shape_checkowidth_for_shape_checks   &&&&&&&&                r7   meta_avg_pool3d_backwardr=    s    
LLKF"X 
QB;1$+a.B;1$+a.B	LL
++c&kV+\ vayBc&kQ&6F1IBc&kQ&6F1IB	LLGT 1:Dw<1$4'!*Dw<1$4'!*D	LL

fK
 
LL55 0A 5*
 jjnGJJrNEjjnGZZ^F0"aS27Bb!YW1&"dB9U#





', ??5;;''r9   c                 H  a  \         P                  ! S P                  ^8H  ;'       g    S P                  ^8H  V 3R l4       S P                  RR \	        V4      ,           p\
        P                  ! S 4      p\         P                  ! VS P                  S P                  VR7      # )r  c                  "   < R S P                    2# )z"Expected 3D or 4D tensor, but got r8  r   s   r7   rl   *meta_adaptive_avg_pool2d.<locals>.<lambda>5      4TZZLAr9   Nr  r  )
r]   ro   r   r   rn   rV   r   r   re   r   )r   output_sizer  r   s   f&  r7   meta_adaptive_avg_pool2drC  1  s    	LL		Q(($))q.A ::cr?U;%77L//5M ;;jj{{#	 r9   c                    a  \         P                  ! S P                  ^8H  ;'       g    S P                  ^8H  V 3R l4       S P                  S P                  RR \        V4      ,           4      # )r  c                  "   < R S P                    2# )z"Expected 4D or 5D tensor, but got r8  r   s   r7   rl   *meta_adaptive_avg_pool3d.<locals>.<lambda>G  rA  r9   Nr  )r]   ro   r   r   r   rn   )r   rB  s   f&r7   meta_adaptive_avg_pool3drG  C  sR    	LL		Q(($))q.A >>$**Sb/E+,>>??r9   c                   a aa S P                   p\        ^V4       F1  o\        P                  ! S P	                  S4      ^ 8  V V3R l4       K3  	  \        P                  ! V^8H  ;'       g    V^8H  V3R l4       \        P                  ! SP
                  S P
                  8H  V V3R l4       \        P                  p\        S4      '       d   \        P                  pSP                  SP                  4      P                  VR7      # )r2   c                  *   < R S P                    RS R2# )z{adaptive_avg_pool2d_backward(): Expected grad_output to have non-zero                       size for non-batch dimensions,  with dimension  being emptyr8  )grad_outr   s   r7   rl   4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>R  s$     66>nn5EEUVWUXXdfr9   c                  "   < R S P                    2# )zBadaptive_avg_pool2d_backward(): Expected 3D or 4D tensor, but got r8  r   s   r7   rl   rM  W  s    TUYU_U_T`ar9   c                  <   < R SP                    RS P                    2# expected dtype z! for `grad_output` but got dtype r   )rL  r   s   r7   rl   rM  [  s    /$**-Nx~~N^_r9   r   )r   r   r]   ro   r   re   r   rj  rg  r   r   rr  )rL  r   r   r   r   s   ff  @r7   "meta__adaptive_avg_pool2d_backwardrR  L  s    ==D1d^MM!q f	
  
LL	TQYa 
LL

hnn$_ ++M++>>$**%((}(EEr9   c                 f    \        V R 4       \        P                  ! V\        P                  R7      # )adaptive_avg_pool3d_backwardr   )!_adaptive_pool_empty_output_checkr]   r   r  r  r   s   &&r7   "meta__adaptive_avg_pool3d_backwardrW  c  s(     &k3QRD0N0NOOr9   c                0    V ^8  d   QhR\         R\        /# )rA   r  r5  r  )rC   s   "r7   rD   rD   j  s     	
 	
6 	
S 	
r9   c                    a aa S P                   p\        ^V4       F2  o\        P                  ! S P	                  S4      ^ 8  VV V3R l4       K4  	  R# )r2   c                  .   < S  R SP                    RS R2# )zc(): Expected grad_output to have non-zero size for non-batch dimensions, but grad_output has sizes rJ  rK  r8  )r5  r  r   s   r7   rl   3_adaptive_pool_empty_output_check.<locals>.<lambda>o  s*    * --8->->,??OPQsR^`r9   N)r   r   r]   ro   r   )r  r5  r   r   s   ff @r7   rU  rU  j  sB    D1d^Q!#	
 r9   c                 0  a a S P                   p\        P                  ! VR9   V 3R l4       \        ^V4       F1  o\        P                  ! S P	                  S4      ^ 8  VV 3R l4       K3  	  \        P                  ! \        V4      ^8H  R 4       ^p^p^ pS P                   ^8X  d   S P	                  ^ 4      pV^,          pS P	                  V^,
          4      pVw  rgS P                   ^8X  d:   WVV3pS P                  V4      p	S P                  V\        P                  R7      p
W3# WEWg3p\        P                  ! S 4      pS P                  V4      P                  VR7      p	S P                  V\        P                  R7      P                  VR7      p
W3# )r  c                  "   < R S P                    2# )z:adaptive_max_pool2d(): Expected 3D or 4D tensor, but got: r8  r  s   r7   rl   *meta_adaptive_max_pool2d.<locals>.<lambda>|      LU[[MZr9   c                  *   < R SP                    RS  R2# )zjadaptive_max_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes rJ  rK  r8  r   r   s   r7   rl   r^         '',{{m3CA3lTr9   c                      R # )zCadaptive_max_pool2d(): internal error: output_size.size() must be 2r3   r3   r9   r7   rl   r^        Ur9   r   r   r  r  )r   r]   ro   r   r   r   r   r   rV   r   rr  )r   rB  r   dimHsizeBsizeDosizeHosizeWr   r   r   r   r   s   f&          @r7   meta_adaptive_max_pool2drk  v  sk    ::D	LLZ 1d^JJqMA	
  
LLKAU
 DEEzzQ

1	JJtax E NFzzQF+	ooi(//)5;;/?|62	33E:ooi(++-+H//)5;;/?BB' C 
 |r9   c                 Z  a a S P                   p\        P                  ! VR9   V 3R l4       \        S R4       \        P                  ! SP                  S P                  8H  V V3R l4       \
        P                  ! S4      pSP                  SP                  4      P                  VR7      # )r  c                  "   < R S P                    2# )zKadaptive_max_pooling2d_backward(): Expected 3D or 4D grad_output, but got: r8  r  s   r7   rl   3meta_adaptive_max_pool2d_backward.<locals>.<lambda>  s    ]^i^o^o]pqr9   adaptive_max_pool2d_backwardc                  <   < R SP                    RS P                    2# rP  r   )r  r   s   r7   rl   ro    s    /%++.OP[PaPaObcr9   r   re  )
r   r]   ro   rU  re   rV   r   r   r   rr  )r  r   r   r   r   s   ff&  r7   !meta_adaptive_max_pool2d_backwardrr    s     D	LLq
 &k3QR	LL{(((c
 //6M??5;;'***GGr9   c                 *  a a S P                   p\        P                  ! VR9   V 3R l4       \        ^V4       F1  o\        P                  ! S P	                  S4      ^ 8  VV 3R l4       K3  	  \        P                  ! \        V4      ^8H  R 4       ^ p^p^ pV^8X  d   S P	                  ^ 4      pV^,          pS P	                  V4      pVw  rgpV^8X  d   WVWx3p	MWEWgV3p	S P                  V	4      p
S P                  V	\        P                  R7      pW3# )r  c                  "   < R S P                    2# )z:adaptive_max_pool3d(): Expected 4D or 5D tensor, but got: r8  r  s   r7   rl   *meta_adaptive_max_pool3d.<locals>.<lambda>  r_  r9   c                  *   < R SP                    RS  R2# )zjadaptive_max_pool3d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes rJ  rK  r8  ra  s   r7   rl   ru    rb  r9   c                      R # )zCadaptive_max_pool3d(): internal error: output_size.size() must be 3r3   r3   r9   r7   rl   ru    rd  r9   r   r%  )r   r]   ro   r   r   r   r   r   )r   rB  r   dimDrg  rh  osizeTri  rj  r   r   r   r   s   f&          @r7   meta_adaptive_max_pool3drz    s    ::D	LLZ 1d^JJqMA	
  
LLKAU
 DEEqy

1	JJtE(FFqyF3	66:	
//)
$Cooiu{{o;G<r9   c                 P    \        V R 4       VP                  VP                  4      # )adaptive_max_pool3d_backward)rU  r   r   )r  r   r   s   &&&r7   !meta_adaptive_max_pool3d_backwardr}    s"     &k3QR??5;;''r9   c                 B    Vf   \        R4      hV P                  V4      # )Nz:cannot repeat_interleave a meta tensor without output_size)rq  r   )repeatsrB  s   &&r7   meta_repeat_interleave_Tensorr    s%    WXX[))r9   c                    V P                   P                  '       g   \        R V P                    24      hVP                   P                  '       g   \        RVP                    24      h\        V P	                  \        V P                   4      4      VP	                  \        VP                   4      4      \        P                  R7      pV# )z!real must be floating point, got z!imag must be floating point, got rS   )re   r  r   r[   rr  r   r   rW   )realimagrA  s   && r7   meta_complexr    s     ::'''@MNN::'''@MNN+DJJ78+DJJ786>>F
 Mr9   
fill_valuec                $    V ^8  d   QhR\         /# )rA   r  rL  )rC   s   "r7   rD   rD     s     

 

c 

r9   c                   \        V 4      R9   d1   V P                  WP                  4       3\        P                  R7      # \        P
                  ! WP                  4       3^V3\        P                  V P                  R7      # )r   r   re   r   )r   mpsr  )r   r   r   r]   r   r  r   )r   r   r  s   &$$r7   nonzero_staticr    sf     411~~tXXZ0

~CC""88:I**;;	
 	
r9   c                    \         P                  ! \        P                  R  4       \         P                  ! V P                  4       V P                  4       3^V P                  4       3\         P                  V P                  R7      # )c                      R # )aY  The register_meta function for torch.nonzero() raises unimplemented by default, as a correct data-independent implementation does not exist. This implementation returns a fake value, assuming all elements of the tensor are non-zero. To enable this registration, please set 'torch.fx.experimental._config.meta_nonzero_assume_all_nonzero' to True.r3   r3   r9   r7   rl   nonzero.<locals>.<lambda>  s     Sr9   r  )	r]   _check_not_implementedr  meta_nonzero_assume_all_nonzeror  r   r   r   r   r   s   &r7   nonzeror    sf     
  22	S 	txxz"	
DJJLjj{{	 r9   c           
      	  a aaaaaaaa \         P                  ! \        S4      R  4       . p\        S4       EF  w  ooSEe   \         P                  ! SP                  \         P
                  \         P                  \         P                  \         P                  39   R 4       SP                  \         P                  \         P                  39   d   SP                  4       p\        V4      o\         P                  ! SSP                  ,           S P                  8*  V 3R l4       \        SP                  4       Fo  o\         P                  ! SP                  S,          S P                  SS,           ,          8H  VVVVV 3R l4       VP                  VP                  ^S4      4       Kq  	  EK{  VP                  S4       EK  VP                  S4       EK  	  Vo\         P                  ! \        S4      S P                  8*  VV 3R l4       ^ RIHp \%        VP&                  ! S!  4      o\        S4      S P                  8  d   SP                  R4       K-  ^ pRpS F+  oV^ 8X  d   Se   ^pK  K  V^8X  d   Sf   ^pK#  K%  Sf   K+   M	  RpV'       g   . p. p\        S4       F.  w  ooSf   K  VP                  S4       VP                  S4       K0  	  \        S4       F.  w  ooSe   K  VP                  S4       VP                  S4       K0  	  S P)                  V4      o Vo. o. o. o\        S4       Fo  w  p	oSfQ   S'       d%   SP                  S P                  V	,          4       K6  SP                  S P                  V	,          4       KZ  \%        SP                  4      oKq  	  VVV3R lp
S P+                  SS,           S,           4      p^ R	IHp V! S P1                  4       ^ 8H  4      '       d   V# V
! S 4      p\2        P4                  ! V4      w  r\%        V4      \%        \        \        V4      4      4      8w  d   \2        P6                  ! VP                  V4      p\2        P8                  ! V4      p\2        P6                  ! V\2        P:                  ! V4      4      pVP=                  VP?                  4       V4      pV# )
c                      R # )z#at least one index must be providedr3   r3   r9   r7   rl   #meta_index_Tensor.<locals>.<lambda>'  s    (Mr9   Nc                      R # )z?tensors used as indices must be long, int, byte or bool tensorsr3   r3   r9   r7   rl   r  /  s    Yr9   c                  "   < R S P                    2# ))too many indices for tensor of dimension rv  r   s   r7   rl   r  6  s    G		{Sr9   c            	      V   < R SP                    RS  RSP                    RSS,            2# )zThe shape of the mask 
 at index z0 does not match the shape of the indexed tensor r8  )r   r   jr  r   s   r7   rl   r  ;  s:    "8ZPQs SJJN**U_`ade`e_f!hr9   c                  <   < R SP                    R\        S 4       R2# )r  z (got r   )r   r   )r   r   s   r7   rl   r  F  s    ;DII;fSQX\NZ[\r9   FTc                   < SS,           S,           p\        V P                  4       4      p^ .\        S4      ,          V\        S4      \        V P                  4      \        S4      ,
          % V P	                  W4      # )z9
This follows restride_src in TensorAdvancedIndexing.cpp
)r   r   r   r   
as_strided)r   r   r   after_shapebefore_shapereplacement_shapes   &  r7   _restride_src(meta_index_Tensor.<locals>._restride_src  sm     00;>t{{}%KL#PSQ
 K
L!C

Oc+6F$FG u..r9   r|  ) r]   ro   r   	enumeratere   r   r   r  r  r   r   r   r   r   r[  selecttorch._refsrs  r   r&   r   r   ro  r}  r   rV   3compute_elementwise_output_logical_to_physical_perm
apply_permr   invert_permr  r   )r   r   rA  r  refsstatehas_contiguous_subspacer9  transposed_indicesr   r  r   r}  restrided_selfpermrY   
perm_shaper~  r  r  r   r   r  r  r  s   ff                @@@@@@@r7   meta_index_Tensorr  %  s   	LLg MN #%Fg&5LL

EIIuzz5::NNY {{uzz5::66--/K""

Ndii/S uzz*A&&A$**QU*;;h h
 MM'..A"67 + e$MM% / '0 G	LLG		!\
 4(('23G
g,
"t E#A:  !aZ}     #'
 #!'*HAu A"))%0 + "'*HAu}A"))%0 + ||D!$ !LK#%(
U= ""4::c?3##DJJsO4 $U[[ 1 )
/ ..(99KG
HCDdjjla'((

 #4(NGGWGD DzT%D	*++%%cii6
66zB
%%j%2C2CD2IJ
nnSXXZ4Jr9   c                 d   R pR pR pR pV! W4      pV
^ ,          '       d0   V P                  VP                  4       4      P                  VR7      pV
^,          '       d0   V P                  VP                  4       4      P                  VR7      pV
^,          '       d   V P                  V4      pWV3# )Nc                 6   \        V 4      p\        V4      pV\        P                  8X  g   V\        P                  8X  d   \        P                  # V\        P                  8X  g   V\        P                  8X  d   \        P                  # \        P                  # rH   )r   r]   rg  r  r   )t1t2fmt1fmt2s   &&  r7   _conv_memory_format6meta_convolution_backward.<locals>._conv_memory_format  sp    $R($R(5&&&$%2E2E*E&&&5)))TU5K5K-K)))&&&r9   r   )r   r   rr  )grad_output_input_weight_bias_sizes_optr   r`  rD  
transposedrG  rF  output_maskbackend_grad_inputbackend_grad_weightbackend_grad_biasr  r   s   &&&&&&&&&&&     r7   meta_convolution_backwardr    s      ' (8M1~~)33FKKMBEE' F 
 1~~*44W\\^DGG' H 
 1~~(22>B5FGGr9   c                  aa SP                  ^4      pSP                  ^4      pV P                  WV34      p \        P                  ! SP	                  4       ^8H  R 4       \        P                  ! SP	                  4       ^8H  R 4       \        P                  ! SP                  ^ 4      SP                  ^ 4      8H  VV3R l4       \        P                  ! SP                  ^4      SP                  ^4      8H  VV3R l4       \        P                  ! V P                  ^ 4      V8H  ;'       d    V P                  ^4      V8H  R 4       V P                  V P                  4       4      # )r2   c                      R # r  r3   r3   r9   r7   rl   meta_addbmm.<locals>.<lambda>  r  r9   c                      R # r  r3   r3   r9   r7   rl   r    r  r9   c                  P   < R S P                  ^ 4       RSP                  ^ 4       2# )z8batch1 and batch2 must have same number of batches, got r   r   r  r  s   r7   rl   r    s,    J6;;WX>JZZ_`f`k`klm`n_opr9   c            
         < R S P                  ^4       RS P                  ^4       RSP                  ^4       RSP                  ^4       R2	# )z#Incompatible matrix sizes for bmm (r<   r   r   r   r  s   r7   rl   r    sL    1&++a.1A6;;q>BR S;;q>"!FKKN#316r9   c                      R # )z.self tensor does not match matmul output shaper3   r3   r9   r7   rl   r    s    @r9   )r   rt  r]   ro   r   r   )r   r  r  r  r  r  r	  s   &ff$$  r7   meta_addbmmr    s    ;;q>D;;q>D;;|$D	LL"$HI	LL"$HI	LLA&++a.(p 
LLA&++a.(	
 
LL		!551!5@ >>$))+&&r9   c                 @    V P                  V P                  4       4      # rH   r   r   )r   r*  kwargss   &&,r7   meta_randint_liker    s    >>$))+&&r9   
grad_scale	found_infc       	         r   a WW#WE3 F,  o\         P                  ! \        S\        4      V3R  l4       K.  	  R# )c                      < R \        S 4       2# z'exponent must be a tensor list but got r   ls   r7   rl   #meta__fused_adam_.<locals>.<lambda>      =d1gYGr9   Nr]   ro   ry   r   )r   gradsexp_avgsexp_avg_sqsmax_exp_avg_sqsstate_stepslrbeta1beta2weight_decayepsamsgradmaximizer  r  r  s   &&&&&&$$$$$$$$$@r7   meta__fused_adam_r    s0    & 8/Oq$G	
 Pr9   c       	            a WW#WE3 F,  o\         P                  ! \        S\        4      V3R  l4       K.  	  R pV! V 4      V! V4      V! V4      V! V4      V! V4      3# )c                      < R \        S 4       2# r  r  r  s   r7   rl   "meta__fused_adam.<locals>.<lambda>+  r  r9   c                 X    V  Uu. uF  p\         P                  ! V4      NK  	  up# u upi rH   r  )tensor_listr,  s   & r7   empty_like_list)meta__fused_adam.<locals>.empty_like_list.  s%    -89[  #[999s   'r  )r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s   &&&&&&$$$$$$$$$ @r7   meta__fused_adamr    si    & 8/Oq$G	
 P: 	!$( r9   c                   a a \         P                  ! S P                  4       ^8H  R 4       \         P                  ! SP                  4       ^8H  R 4       \         P                  ! S P                  \         P                  \         P
                  39   V 3R l4       \         P                  ! SP                  \         P                  J V3R l4       \         P                  ! S P                  ^4      SP                  ^ 4      8H  V V3R l4       S P                  S P                  ^ 4      SP                  ^4      3\         P                  R7      # )rA   c                      R # )za must be a 2D tensorr3   r3   r9   r7   rl   meta__int_mm.<locals>.<lambda>=      '>r9   c                      R # )zb must be a 2D tensorr3   r3   r9   r7   rl   r  >  r  r9   c                  "   < R S P                    2# )z'expected self to be int8 or uint8, got r   )r5   s   r7   rl   r  A  s    9!''Cr9   c                  "   < R S P                    2# )zexpected mat2 to be int8, got r   )r6   s   r7   rl   r  E  s    0	:r9   c            
         < R S P                  ^ 4       RS P                  ^4       RSP                  ^ 4       RSP                  ^4       R2	# )z'Incompatible matrix sizes for _int_mm (r<   r   r   r   r4   s   r7   rl   r  I  sH    5affQi[!&&) M66!9+Qqvvayk,r9   r   )	r]   ro   r   re   r  r  r   r   r  r4   s   ffr7   meta__int_mmr  :  s     
LLA>?	LLA>?	LL	EJJ,,C 
LL	5::: 
LL	q	QVVAY	
 ;;q	166!9-U[[;AAr9   c                   a  \         P                  ! S P                  4       ^8H  R 4       \         P                  ! S P                  \         P                  J V 3R l4       S P                  ^ 4      pS P                  ^4      ^,          pS P                  V^,          W1^,          ,          ^ V^,          3\         P                  R7      # )rA   c                      R # zw must be a 2D tensorr3   r3   r9   r7   rl   2meta__convert_weight_to_int4pack.<locals>.<lambda>S  r  r9   c                  "   < R S P                    2# r  r   r  s   r7   rl   r  V      .qwwi8r9   r   )r]   ro   r   re   r  r   r   r  r  inner_k_tilesr  r  s   f&  r7    meta__convert_weight_to_int4packr  Q  s    	LLA>?	LL	5;;8 	
q	A	q	AA;;F"$%Q		
 kk   r9   c                 V  a  \         P                  ! S P                  4       ^8H  R 4       \         P                  ! S P                  \         P                  J V 3R l4       S P                  ^ 4      pS P                  ^4      pS P                  W#^,          3\         P                  R7      # )rA   c                      R # r  r3   r3   r9   r7   rl   :meta__convert_weight_to_int4pack_for_cpu.<locals>.<lambda>g  r  r9   c                  "   < R S P                    2# zexpected w to be int32, got r   r  s   r7   rl   r  j  r  r9   r   )r]   ro   r   re   r  r   r   r  r  s   f&  r7   (meta__convert_weight_to_int4pack_for_cpur  e  s}    	LLA>?	LL	5;;8 	
q	A	q	A;;	
Fkk   r9   c                   a aa \         P                  ! S P                  4       ^8H  R 4       SP                  P                  R8X  d   ^M^o\         P                  ! SP                  4       S8H  V3R l4       \         P                  ! S P
                  \         P                  \         P                  \         P                  39   V 3R l4       \         P                  ! SP
                  \         P                  J V3R l4       SP                  P                  R8X  d   SP                  ^ 4      MSP                  ^ 4      ^,          pS P                  S P                  ^ 4      VS P
                  R7      # )rA   c                      R # zx must be a 2D tensorr3   r3   r9   r7   rl   *meta__weight_int4pack_mm.<locals>.<lambda>v  r  r9   r  c                     < R S  R2# )zw must be a zD tensorr3   )expected_dims   r7   rl   r  x  s    Lh2Wr9   c                  "   < R S P                    2# r  r   r  s   r7   rl   r  {      5aggY?r9   c                  "   < R S P                    2# r  r   r  s   r7   rl   r    r  r9   r   )r]   ro   r   r@  r   re   r  r  r  r  r   r   )r<   r  r  r  dim_nr  s   ff&& @r7   meta__weight_int4pack_mmr  t  s    	LLA>?**e31L	LLL(*WX	LL	EMM5==%..AA? 
LL	5;;8 ++u4AFF1I!&&)a-E;;qvvay%qww;77r9   c                 6  a a \         P                  ! S P                  4       ^8H  R 4       \         P                  ! SP                  4       ^8H  R 4       \         P                  ! S P                  \         P                  \         P
                  \         P                  39   V 3R l4       \         P                  ! SP                  \         P                  J V3R l4       S P                  S P                  ^ 4      SP                  ^ 4      S P                  R7      # )rA   c                      R # r
  r3   r3   r9   r7   rl   2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>  r  r9   c                      R # r  r3   r3   r9   r7   rl   r    r  r9   c                  "   < R S P                    2# r  r   r  s   r7   rl   r    r  r9   c                  "   < R S P                    2# r  r   r  s   r7   rl   r    r  r9   r   )
r]   ro   r   re   r  r  r  r  r   r   r  s   ff&&r7    meta__weight_int4pack_mm_for_cpur        	LLA>?	LLA>?	LL	EMM5==%..AA? 
LL	5;;8 ;;qvvay!&&)177;;;r9   c                 6  a a \         P                  ! S P                  4       ^8H  R 4       \         P                  ! SP                  4       ^8H  R 4       \         P                  ! S P                  \         P                  \         P
                  \         P                  39   V 3R l4       \         P                  ! SP                  \         P                  J V3R l4       S P                  S P                  ^ 4      SP                  ^ 4      S P                  R7      # )rA   c                      R # r
  r3   r3   r9   r7   rl   ;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>  r  r9   c                      R # r  r3   r3   r9   r7   rl   r    r  r9   c                  "   < R S P                    2# r  r   r  s   r7   rl   r    r  r9   c                  "   < R S P                    2# r  r   r  s   r7   rl   r    r  r9   r   )
r]   ro   r   re   r  r  r  r  r   r   )r<   r  r  qScaleqZeross   ff&&&r7   )_weight_int4pack_mm_with_scales_and_zerosr#    r  r9   c                <    V ^8  d   QhR\         R\         R\         /# )rA   r5   r6   rB   rL  )rC   s   "r7   rD   rD     s!     " "3 "3 "3 "r9   c                 <    W,           ^,
          V,          V,          # r1   r3   r4   s   &&r7   kai_roundupr&    s    UQY1!!r9   c                 :  a	a
aaaaaaaaa V ^8X  d   W28X  d)   ^p^p^p^o^o^oR oVVVV3R loV3R lpV! WWEV4      # V^ ,          ^ 8X  dK   W#,          ^ 8X  d<   ^p^p^p^o^o^o^ o	V	VV3R lpV	V
VVVVV3R loR o
V	3R loV	3R loV! WWEWc4      # R	# R	# R	# )
r  c                 <    \        W,          ^4      p\        W4      # )r  r&  )r  krsrkr_sr_roundedup4s   &&& r7   kai_k_roundedup3get_kai_packed_weight_size.<locals>.kai_k_roundedup  s     $/rw#: "177r9   c                    < S! WV4      pV^,          ^ 8w  d   \        RV 24      hVV^,          S,           S,           S,           ,          # )rA   zk_internal must be even, got r  )	r  nrr*  r+  
k_internalr-  kai_num_bytes_biaskai_num_bytes_multiplier_rhskai_num_bytes_sum_rhss	   &&&& r7   9kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0  s^     -QB7
Nq((+H)UVV1_23+, )) r9   c                 H   < \        W4      V,          pVS! WW44      ,          # rH   r)  )r  r  r0  r*  r+  num_rowsr5  s   &&&&& r7   7kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0[get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0  s0     'q-3 Orr9   c                   < WS,          ^ 8w  d   \        RV RV R24      hVS	,          ^ 8w  d   \        RV RS	 R24      hVS,          ^ 8w  d   \        RV RS R24      h\        W4      V,          pVS! WW4V4      ,          # r   bl (z) must be divisible by kr (r   znr (z+) must be divisible by kai_nr_multiple_of (+) must be divisible by kai_bl_multiple_of (r   r&  )
r  r  r0  r*  r+  blr8  kai_bl_multiple_of;kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0kai_nr_multiple_ofs
   &&&&&& r7   9kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0  s     G>(4t3NrdRS)TUU++1(rd"MN`Maabc  ++1(rd"MN`Maabc  'q-3 Qrrr9   c                 *  < WB,          ^ 8w  d   \        RV RV R24      hVS
,          ^ 8w  d   \        RV RS
 R24      hVS,          ^ 8w  d   \        RV RS R24      hS	! 4       pS! W4      pS! WE4      pVWv,          S,           S,           ,          # r<  r  )r  r0  r*  r+  r@  num_bytes_multiplier_rhsnum_blocks_per_rownum_bytes_per_blockrA  #kai_get_bf16_datatype_size_in_bytesrC  kai_num_blocks_per_rowr2  kai_num_bytes_per_blockr4  s   &&&&&   r7   rB  _get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0  s     G>(4t3NrdRS)TUU++1(rd"MN`Maabc  ++1(rd"MN`Maabc 
 ,O+P(%;A%B"&='# (=+,() r9   c                      ^# rA   r3   r3   r9   r7   rJ  Gget_kai_packed_weight_size.<locals>.kai_get_bf16_datatype_size_in_bytes  s    r9   c                 h   < VS,          ^ 8w  d   \        RV RS R24      h\        W4      V,          # r   r=  r>  r   r?  )r  r@  rA  s   &&r7   rK  :get_kai_packed_weight_size.<locals>.kai_num_blocks_per_row  sF    ++1(rd"MN`Maabc  #1)R//r9   c                 d   < V S,          ^ 8w  d   \        RV  RS R24      hV ^,          V,           # rR  r  )r@  rG  rA  s   &&r7   rL  ;get_kai_packed_weight_size.<locals>.kai_num_bytes_per_block  sF    ++1(rd"MN`Maabc  a#;;;r9   Nr3   )n_bitsr4  K	groupsizekai_nrkai_krkai_srr9  rD  rA  rJ  rB  r5  r-  rC  rK  r2  r3  rL  r4  s   &&&&     @@@@@@@@@@@r7   get_kai_packed_weight_sizer\    s    {>FFF$%!+,(!"8 
 Kff  ^q Q]a%7FFF$%!!"!"!#. :0< Mff ] &8 _ r9   c                2    V ^8  d   QhR\         R,          /# )rA   r  Nr_  )rC   s   "r7   rD   rD   +  s     D D!'$Dr9   c                   a  \         P                  ! S P                  \         P                  J V 3R  l4       \         P                  P
                  P                  4       '       d   W48X  d    VP                  \         P                  8X  gA   W48  dr   V^ ,          ^ 8X  dd   WC,          ^ 8X  dW   VP                  \         P                  8X  d8   \        ^WTV4      pS P                  \        V4      \         P                  R7      # S P                  4       VP                  4       ,           pVe   WbP                  4       ,          pS P                  V\         P                  R7      # )c                  "   < R S P                    2# r  r   )weightss   r7   rl   2meta__dyn_quant_pack_4bit_weight.<locals>.<lambda>0  s    .w}}o>r9   r   )r]   ro   re   r  r  kleidiair  ra   r  r\  r   r   r   )r`  scales_zerosr  
block_sizein_featuresout_featurespacked_weight_sizes   f&&&&& r7    meta__dyn_quant_pack_4bit_weightrh  *  s    
LL$> ~~++--		"|'9'9U[['H$R1$(A-""enn4 8|*
   %7!8 LL <+=+=+??jjl*/u{{CCr9   c                   a aa \         P                  ! S P                  4       ^8H  R 4       \         P                  ! S P                  \         P                  8H  ;'       g+    S P                  \         P
                  8H  ;'       d    SS8H  VVV 3R l4       S P                  ^ 4      pS P                  WTS P                  R7      # )rA   c                      R # )zinput must be a 2D tensorr3   r3   r9   r7   rl   -meta__dyn_quant_matmul_4bit.<locals>.<lambda>M  s    )Dr9   c                  .   < R SP                    RS  RS 2# )zPexpected input to be f32 or bf16 (bf16 requires block_size == in_features), got z with block_size=z and in_features=r   )rd  re  inps   r7   rl   rk  Q  s%    99+.zl:KK=Zr9   r   )r]   ro   r   re   r  r  r   r   )rm  packed_weightsrd  re  rf  r4  s   f&ff& r7   meta__dyn_quant_matmul_4bitro  E  s     
LLa!DE	LL	emm	# 	G 	GII'EEJ+,E	
 	A==		=::r9   c                 6  a a \         P                  ! S P                  4       ^8H  R 4       \         P                  ! S P                  \         P                  \         P
                  \         P                  39   V 3R l4       \         P                  ! SP                  4       ^8H  R 4       \         P                  ! SP                  \         P                  J V3R l4       S P                  S P                  ^ 4      SP                  ^ 4      S P                  R7      # )rA   c                      R # r
  r3   r3   r9   r7   rl   *meta__weight_int8pack_mm.<locals>.<lambda>\  r  r9   c                  "   < R S P                    2# r  r   r  s   r7   rl   rr  _  r  r9   c                      R # r  r3   r3   r9   r7   rl   rr  a  r  r9   c                  "   < R S P                    2# )zexpected w to be int8, got r   r  s   r7   rl   rr  d  s    -aggY7r9   r   )
r]   ro   r   re   r  r  r  r  r   r   )r<   r  q_scaless   ff&r7   meta__weight_int8pack_mmrw  Z  s    	LLA>?	LL	EMM5==%..AA? 
LLA>?	LL	5::7 ;;qvvay!&&)177;;;r9   c                   a aa \         P                  ! S P                  4       ^8  V 3R l4       \         P                  ! SP                  4       ^8  V3R l4       \         P                  ! S P                  R	4      SP                  R	4      8H  V V3R l4       \         P                  ! \        P
                  ! S P                  4      V 3R l4       \         P                  ! \        P
                  ! SP                  4      V3R l4       \         P                  ! V^ 8  R 4       \         P                  ! SR
9   V3R l4       S P                  R4      pSP                  R4      pS P                  RR pSP                  RR p\        \         P                  ! Wg4      4      pVP                  WE.4       S P                  V4      # )rA   c                  ,   < R S P                  4        R2# )z1cdist only supports at least 2D tensors, X1 got: r  r  x1s   r7   rl   $meta_cdist_forward.<locals>.<lambda>m      CBFFH:QOr9   c                  ,   < R S P                  4        R2# )z1cdist only supports at least 2D tensors, X2 got: r  r  x2s   r7   rl   r|  q  r}  r9   c                  P   < R S P                  R4       RSP                  R4       2# )z4X1 and X2 must have the same number of columns. X1: z X2: r   r   )r{  r  s   r7   rl   r|  u  s*    Frwwr{mSXY[Y`Y`acYdXefr9   c                  "   < R S P                    2# )z3cdist only supports floating-point dtypes, X1 got: r   rz  s   r7   rl   r|  y      EbhhZPr9   c                  "   < R S P                    2# )z3cdist only supports floating-point dtypes, X2 got: r   r  s   r7   rl   r|  }  r  r9   c                      R # )z)cdist only supports non-negative p valuesr3   r3   r9   r7   rl   r|    s    !Lr9   Nc                     < R S  2# )z(possible modes: None, 0, 1, 2, but was: r3   )compute_modes   r7   rl   r|    s    :<.Ir9   r   )Nr   r2   rA   r  )r]   ro   r   r   rV   is_float_dtypere   r   r   broadcast_shapesextendr   )	r{  r  r  r  r1r2batch_tensor1batch_tensor2r  s	   ff&f     r7   meta_cdist_forwardr  i  sF   	LL
AO 
LL
AO 
LL
rwwr{"f 
LLRXX&P 
LLRXX&P 
LLaLM	LL'I 
B	BHHSbMMHHSbMM..}LML!<<%%r9   c                 d   VP                   R,          pVP                   R,          pVP                   R,          pVP                   RR pVP                   RR p	\        \        P                  ! W4      4      p
V
P	                  4       pVP                  We.4       \        P                  ! V
4      pV^ 8X  g   V^ 8X  g   V^ 8X  g   V^ 8X  d   \        P                  ! V4      # V\        VP                   4      8w  d   VP                  V4      p\        P                  ! V\        P                  R7      # )r2   Nr   r   r  )r   r   r]   r  copyr  mathprod
zeros_likert  r   r   )r  r{  r  r  cdistc1r  r  r  r  r	  tensor1_expand_sizebatch_products   &&&&&        r7   meta_cdist_backwardr    s     
"B	"B	"BHHSbMMHHSbMM 6 6} TU.335x(II23M	Qw"'R1W(:##d288n,YY*+Be.E.EFFr9   c	                   a aaaaa \         P                  ! SP                  \         P                  \         P                  39   V3R  l4       \         P                  ! SP                  \         P                  \         P                  39   V3R l4       \         P                  ! \
        P                  ! S P                  4      V 3R l4       SP                  ^ 4      p	V'       d%   \         P                  ! V	^8  R 4       V	^,          p	S P                  V	S P                  ^4      4      p
Se   \         P                  ! V\        8H  R 4       \         P                  ! SP                  ^8H  V3R l4       \         P                  ! SP                  4       SP                  4       8H  VV3R l4       V3R loR oVV3R	 lp\        S4      R
8w  d   SP                  SP                  ^ 4      4      pSP                  SP                  4       4      pV\        8X  d$   SP                  V	S P                  ^4      4      pEMSP                  ^ 4      pMV! S SW4      pV\        \        39   g	   V'       g"   SP                  SP                  ^ 4      4      pMSP                  ^ 4      pSP                  V	4      pSP                  ^ ,          pV\        8X  dQ   V'       d%   \         P                  ! V^8  R 4       V^,          pSP                  VS P                  ^,          4      pMSP                  VP                  4       4      pWW3# )c                  "   < R S P                    2# )z(expected indices to be long or int, got r   )r   s   r7   rl   $meta_embedding_bag.<locals>.<lambda>      :7==/Jr9   c                  "   < R S P                    2# )z(expected offsets to be long or int, got r   )r  s   r7   rl   r    r  r9   c                  "   < R S P                    2# )z/expected weight to be floating point type, got r   )r  s   r7   rl   r    s    A&,,Pr9   c                      R # z1include_last_offset: numBags should be at least 1r3   r3   r9   r7   rl   r    s    Gr9   c                      R # )z@embedding_bag: per_sample_weights only supported with mode='sum'r3   r3   r9   r7   rl   r    s    Vr9   c                  $   < R S P                    R2# )z1expected per_sample_weights to be 1D tensor, got r  rv  )per_sample_weightss   r7   rl   r    s    GHZH_H_G``abr9   c                  N   < R SP                  4        RS P                  4        R2# )z%expected per_sample_weights.numel() (z$ to be the same as indices.numel() (r   r   )r   r  s   r7   rl   r    s/    78J8P8P8R7S T66=mmo5FaIr9   c                 N   < S! WV4      ;'       d    VP                  ^ 4      ^8H  # r  r   )ru  r!  r  padding_idxis_fast_path_index_selects   &&&&r7   is_fast_path_index_select_scale;meta_embedding_bag.<locals>.is_fast_path_index_select_scale  s)    %c;?XXELLQROWXDX	
r9   c                    V P                   \        P                  8H  ;'       g&    V P                   \        P                  8H  ;'       d>    V P	                  ^4      ^8H  ;'       d"    VP	                  ^4      ^8H  ;'       d    V^ 8  # r1   )re   r]   ra   r_   r   )ru  r  r  s   &&&r7   r  5meta_embedding_bag.<locals>.is_fast_path_index_select  st    YY%++%@@ejj)@    

1"   a A%    a		
r9   c                 0   < Ve
   S! WW#4      # S! WV4      # rH   r3   )ru  r!  r  r  r  r  s   &&&&r7   is_fast_path(meta_embedding_bag.<locals>.is_fast_path  s#    23vSS,S+FFr9   r   c                      R # r  r3   r3   r9   r7   rl   r    s    Or9   )r]   ro   re   r   r   rV   r  r   r   MODE_SUMr   r   r   MODE_MAX	MODE_MEANr   )r  r   r  scale_grad_by_freqr  sparser  include_last_offsetr  num_bagsr  r  
offset2bagbag_sizemax_indicesfast_path_sumnumBagsr  r  s   fff&&&f&&        @@r7   meta_embedding_bagr    sw    
LL%**eii00J 
LL%**eii00J 
LLV\\*P
 ||AHMG	
 	AhA7F%HV	
 	##q(b	
 	$$&'--/9	



G 7u$&&w||A7
$$W\\^48!++Hfkk!nEK!++A.K$V-?UIx(( **7<<?;J **1-J$$X.--"8"qLO 1!++GV\\!_EK!++HMMO<Kx44r9   c                     \        WV.VO5!  w  rErg\        V4      R 8X  d    VP                  VP                  4       4      pWEWg3# )r   )r  r   r   r   )r  r   r  rX   r  r  r  r  s   &&&*    r7   meta_embedding_bag_forward_onlyr    sN    0B1#'1-F 7u$$$W\\^4x44r9   c                     V'       d   V# V P                   P                  '       g   V P                   P                  '       d   V P                   # V'       d   \        P                  # V P                   # rH   )re   r  r   r]   r   )r   re   promote_int_to_longs   &&&r7   _get_reduction_dtyper    sJ    {{$$$(>(>(>{{	zz;;r9   c                    \        WR R7      p\        P                  ! V P                  V4      p\	        WV4      pV P                  WTR7      # )T)r  r   )r  rV   r  r   r  r   )r   r9  r  re   r  r  s   &&&$  r7   meta_nansumr    sC     ($OLT2D+EAL??<?<<r9   c           	          \         P                  ! V P                  \        \	        V P                  4       4      4      4      pV P                  V4      # rH   )rV   r;  r   rn   r   r   r   )r   r  s   & r7   meta_medianr  (  s<    77U5-.L ??<((r9   c                    \        V 4      R 8X  d   \        P                  ! R4       \        P                  ! V P                  V34      p\        WV4      pV P                  V4      V P                  V\        P                  R7      3# )r   zmedian CUDA with indices outputr   )	r   rV   alert_not_deterministicr  r   r  r   r]   r   )r   r   r  r  s   &&& r7   meta_median_mode_dimr  0  sn     5V#%%&GH


u{{SF
3C+E@L%EJJ7 r9   c                     V # rH   r3   r   s   &r7   meta_logical_not_r  F  r  r9   c                   aa \         P                  ! \        V4      V P                  4       8  R  4       \	        V4       F%  w  oo\         P                  ! S^ 8  VV3R l4       K'  	  \        V4      V P                  4       ,
          pRV,          \        V P                  4      ,           p\        \        V4      4       Uu. uF  qCV,          W,          ,          NK  	  ppV P                  V4      # u upi )c                      R # )zZNumber of dimensions of repeat dims can not be smaller than number of dimensions of tensorr3   r3   r9   r7   rl   meta_repeat.<locals>.<lambda>O  s    lr9   c                     < R S RS  2# )z"Repeats cannot be negative, found r  r3   )r   reps   r7   rl   r  T  s    8ZsKr9   r1   )	r]   ro   r   r   r  rn   r   r   r   )r   r  num_new_dimensionspadded_sizer   target_sizer  s   &&  ` @r7   meta_repeatr  K  s    	LLG
"l G$31HK	
 % W
2++eDJJ.??K8=c'l8KL8K1q>GJ..8KKL>>+&& Ms   C3c                     V # rH   r3   r   s   &r7   
meta_zero_r  _  r  r9   c                     \        V\        P                  4      '       d!   \        V P                  VP                  4       V # rH   )ry   r]   r   rq   r   r   r   s   &&r7   meta_binop_inplacer  d  s,     %&&

EKK8Kr9   c                4   R pR pR pV! V 4      '       d   V! V4      '       d   \        R4      hV! V 4      '       d   V! V4      '       g   \        R4      h\        V\        P                  4      '       d!   \	        V P
                  VP
                  4       V # )a  
Some checks for inplace ops.
Checks for promotion rules for some dtypes.
int.add/sub_(float) and bool.add/sub_(others) are rejected.
Promoting in these in-place operations would require reallocating
and copying over elements, hence not allowed.
Checks for alpha param.
c                     \        V \        4      '       d!   \        P                  ! V P                  4      # \        V \
        4      # rH   )ry   r   rV   r  re   r   r|   s   &r7   is_integeric.meta_binop_inplace_alpha.<locals>.is_integeric  s1    c:&&))#))44c7++r9   c                     \        V \        4      '       d!   \        P                  ! V P                  4      # \        V \
        4      # rH   )ry   r   rV   r  re   r   r  s   &r7   
is_floatic,meta_binop_inplace_alpha.<locals>.is_floatic  s1    c:&&''		22c9--r9   c                     \        V \        4      '       d!   \        P                  ! V P                  4      # \        V \
        4      # rH   )ry   r   rV   is_boolean_dtypere   r   r  s   &r7   is_booleanic.meta_binop_inplace_alpha.<locals>.is_booleanic  s1    c:&&))#))44c8,,r9   z]Promotion of int.add/sub_(float) in in-place ops are not possible due to element size change.z_Promotion of book.add/sub_(others) in in-place ops are not possible due to element size change.)rq  ry   r]   r   rq   r   )r   r   r  r  r  r  s   &&&   r7   meta_binop_inplace_alphar  u  s    $,.- Dj//k
 	

 D,u"5"5m
 	
 %&&

EKK8Kr9   c                 8    \        W\        P                  R 7      # r  r[   r   rW   r   r   r  s   &&&r7   meta_binop_alphar    s     $C$K$K r9   c                 8    \        V \        P                  R 7      # r  r  )r   r  s   &,r7   
meta_roundr    s    <DD r9   c                   a aa \         P                  ! \        P                  ! SP                  4      V V3R  l4       \        S\         P                  4      '       d=   \         P                  ! \        P                  ! SP                  4      V V3R l4       R# \         P                  ! \        S\        4      V V3R l4       R# )c                  &   < S  R SP                    2# )z7: Expected input tensor to have an integral dtype. Got r   )rC  r   s   r7   rl   #shift_dtype_check.<locals>.<lambda>  s    7)RSWS]S]R^_r9   c                  &   < S  R SP                    2# )z6: Expected shift value to have an integral dtype. Got r   rC  r  s   r7   rl   r    s    wiUVYV_V_U`ar9   c                     < S  R S 2# )z): Expected shift value to be an int. Got r3   r  s   r7   rl   r    s    wiHNr9   N)r]   ro   rV   r  re   ry   r   r   )rC  r   r  s   fffr7   shift_dtype_checkr    ss    	LLtzz*_ #u||$$""399-a	

 	sG$N	
r9   c                 P    \        R W4       \        W\        P                  R7      # )rshiftr  r  r[   r   rW   r  s   &&r7   meta_rshiftsr    %    h,$C$K$K r9   c                 P    \        R W4       \        W\        P                  R7      # )lshiftr  r  r  s   &&r7   meta_lshiftsr    r  r9   c                 8    V P                  V P                  4      # rH   r  r   s   &r7   	meta_zeror     s    >>$**%%r9   c                     V # rH   r3   r   r  s   &&r7   
meta_fill_r    r  r9   c                 .    \         P                  ! V 4      # rH   r  r  s   &&r7   	meta_fillr        D!!r9   c                     V # rH   r3   r   s   &r7   
meta_relu_r    r  r9   c                $    V ^8  d   QhR\         /# r@   r_  )rC   s   "r7   rD   rD     s      F r9   c                 8    \        W\        P                  R 7      # r  r  r  s   &&&r7   meta__add_relur    s     $C$K$K r9   c                 .    \         P                  ! V 4      # rH   r  r   noiselowerrP  rn  r  s   &&&&&&r7   meta_rrelu_with_noiser    s    
 D!!r9   c                 Z    \         P                  ! V 4      \         P                  ! V4      3# rH   r  r  s   &&&&&&r7    meta_rrelu_with_noise_functionalr    s%     D!5#3#3E#:::r9   c                     V # rH   r3   )r   r  rP  rn  r  s   &&&&&r7   meta_rrelu_with_noise_r  	  s	     Kr9   c                 .    \         P                  ! V 4      # rH   r  r   r   r   
accumulates   &&&&r7   meta_index_putr    r  r9   c                 F    \        V P                  VP                  4       V # rH   rq   r   )r   r)  values   &&&r7   meta_masked_fill_r    s    DJJ

3Kr9   c                     V P                  V P                  4       4      P                  \        P                  ! V 4      R 7      pV# r   )r   r   rr  rV   r   )r   r)  r!  masked_scales   &&& r7   meta__masked_scaler    s<    >>$))+.1111$7 2 L r9   c                    a a \         P                  ! VP                  \         P                  \         P                  39   R  4       \         P                  ! S P                  SP                  8H  V V3R l4       S # )c                      R # )zMask must be bool or uint8r3   r3   r9   r7   rl   &meta_masked_scatter_.<locals>.<lambda>&  s    9Ur9   c                  <   < R S P                    RSP                    2# )zEmasked_scatter: expected self and source to have same dtypes but got r   r   )r   r  s   r7   rl   r"  *  s      **U6<<.:r9   )r]   ro   re   r   r  )r   r)  r  s   f&fr7   meta_masked_scatter_r$  #  sU    	LL

uzz5;;//1U 
LL

fll"	:
 Kr9   c                     \        W4      w  r\        P                  ! V \        P                  R 7      p\	        W1V4      # r   )r&   r]   r   r   r$  )r   r)  r  r  s   &&& r7   meta_masked_scatterr&  0  s5     "$-JDd%2I2IJFf55r9   c                 $    V P                  V4      # rH   r  )r   r)  r  s   &&&r7   meta_masked_scatter_backwardr(  8  s    >>%  r9   c                     V # rH   r3   r  s   &&&&r7   meta_index_put_r*  =  r  r9   c           	        a aaaaaa ^ RI HpHp \        P                  ! S P                  4       ^8H  R 4       \        P                  ! SP                  4       ^8H  R 4       \        P                  ! S P                  SP                  8H  V V3R l4       S P                  4       pSP                  4       oV^ ,          oV^,          oV^,          pS^,          p	SW3o\        P                  ! V! V! S^ ,          S4      V! S^,          S4      4      VVV3R l4       V'       d   S P                  \        P                  8H  ;'       g&    S P                  \        P                  8H  ;'       d    V\        P                  8H  p
\        P                  ! VS P                  8H  ;'       g    T
R 4       SP                  S4      P                  V4      pMSP                  S4      pV'       g_   Se[   \        P                  ! SP                  4       ^8H  R 4       \        P                  ! V! SP                  4       S4      VV3R l4       V# )	r   )sym_andr  c                      R # r  r3   r3   r9   r7   rl   )common_meta_baddbmm_bmm.<locals>.<lambda>E  r  r9   c                      R # r  r3   r3   r9   r7   rl   r.  F  r  r9   c                  <   < R S P                    RSP                    2# )zexpected scalar type z but found r   r  s   r7   rl   r.  I  s    '~[Or9   c            	      >   < R S RS RS ^ ,           RS ^,           R2	# r  r3   r  s   r7   rl   r.  W  s3    RSURV
l<?*;2l1o=NbRr9   c                      R # )zfout_dtype only supported for torch.float32 output with float16/bfloat16 inputs or same as input dtypesr3   r3   r9   r7   rl   r.  `  s    |r9   c                      R # )zself must be a 3D tensorr3   r3   r9   r7   rl   r.  h  s    6Pr9   c                  0   < R S  RSP                  4        2# )z*Expected an input tensor shape with shape z but got shape: r   )rB  self_baddbmms   r7   rl   r.  k  s    @M]^j^o^o^q]rsr9   )ro  r,  r  r]   ro   r   re   r   r  r  r  r   rr  )r  r  is_bmmr5  r  r,  r  r  res_rowsres_colssupported_out_dtyper  r  r  r  rB  s   ff&f&       @@@@r7   common_meta_baddbmm_bmmr:  B  s   E	LL"$HI	LL"$HI	LL$O
 ;;=L;;=L	aB#AAHAHx*K	LL|A+VLOEU-VW	R
 LLEMM)KKV\\U^^-K) )5==( 	 	%<<)<|	
 !!+.11)< !!+.l.\%%'1,.PQ<$$&4s	

 Mr9   c                     \        WR 4      # Tr:  )r   r  s   &&r7   meta_bmmr>  q  s     #4t44r9   c                     \        WR VR7      # )T)r  r=  )r   r  r  s   &&&r7   meta_bmm_dtyper@  w  s     #4tyIIr9   c                     W,          pW,          pV^ 8w  d)   \        V^ 8  4      \        V^ 8  4      8w  d
   V^,          pV# r  )r   )r<   r=   qr  s   &&  r7   div_rtnrC  }  s>    	A	A 	Av4A;$q1u+-	QHr9   c                    \        V V,           V,           WQ^,
          ,          ,
          ^,
          V'       d
   V^,
          M^ ,           V4      ^,           pV'       d%   V^,
          V,          W,           8  d
   V^,          pV# r1   )rC  )	inputSize
kernelSizerp  rq  r   rD  r  
outputSizes   &&&&&&& r7   pooling_output_shape_pad_lrrH    s     	 q.)* 	
 'vzA/ 	
 		  Nf$	(99!OJr9   c           	        aaa \         P                  ! V^ 8g  R 4       \         P                  ! S^ 8  V3R l4       \         P                  ! SS^,
          S,          ^,           ^,          8*  VVV3R l4       \        V SSSVSV4      # )r   c                      R # )zstride should not be zeror3   r3   r9   r7   rl   &pooling_output_shape.<locals>.<lambda>  s    &Ar9   c                     < R S  2# )z'pad must be non-negative, but got pad: r3   pads   r7   rl   rK    s    %LSE#Rr9   c                     < R S RS RS  2# )zApad should be at most half of effective kernel size, but got pad=z, kernel_size=z and dilation=r3   )rD  rF  rN  s   r7   rl   rK    s"    OPSu U%,nXJ@r9   )r]   ro   rH  )rE  rF  rN  r   rD  r  s   &ff&f&r7   r  r    ss    	LL1AB	LLRS	LLa8+a/A55	
 ':sC9 r9   c           	      *  a aaaaaaaaa	a
aaaa S P                  4       pS	o\        P                  ! S^ 8  ;'       d    S^ 8  VV3R l4       \        P                  ! S^ 8  ;'       d    S^ 8  VV3R l4       \        P                  ! S^ 8  ;'       d    S^ 8  VV3R l4       S P                  ^4      ^ 8g  ;'       d    S P                  ^4      ^ 8g  pV\        P                  8X  dF   \        P                  ! V^8H  ;'       d    T;'       d    S P                  ^4      ^ 8g  V 3R l4       Mw\        P                  ! V^8H  ;'       d'    S P                  ^ 4      ^ 8g  ;'       d    T;'       g,    V^8H  ;'       d    T;'       d    S P                  ^4      ^ 8g  V 3R l4       \        P                  ! S^,          S8  ;'       d    S^,          S8  VVVV3R l4       \        P                  ! S^8  ;'       d    S^8  V
VV	VVV3R l4       R# )	r   c                     < R S  RS 2# )z5kernel size should be greater than zero, but got kH: , kW: r3   )r  r  s   r7   rl   $pool2d_shape_check.<locals>.<lambda>  s    Gt6RTQUVr9   c                     < R S  RS 2# )z0stride should be greater than zero, but got dH: , dW: r3   )r  r   s   r7   rl   rS    s    B2$fRDQr9   c                     < R S  RS 2# )z9dilation should be greater than zero, but got dilationH: , dilationW: r3   )	dilationH	dilationWs   r7   rl   rS    s    KI;Vcdmcnor9   c                  *   < R S P                  4        2# )zExpected 4D (batch mode) tensor expected for input with channels_last layout with optional 0 dim batch size for input, but got: r   r  s   r7   rl   rS    s     CCH::<.Rr9   c                  *   < R S P                  4        2# )zYExpected 3D or 4D (batch mode) tensor with optional 0 dim batch size for input, but got: r   r  s   r7   rl   rS    s    opupzpzp|o}~r9   c                      < R S RS RS RS  2# )zKpad should be smaller than or equal to half of kernel size, but got padW = z	, padH = z, kW = z, kH = r3   )r  r  r  r  s   r7   rl   rS    s$     ygbT>r9   c                  .   < R S RS  RS RS RS RS R2# zGiven input size: (r<   z). Calculated output size: (z). Output size is too smallr3   )r  r  r  r	  r  r  s   r7   rl   rS    s8    %k]!K=* N$$0><.+ O##r9   N)r   r]   ro   r   rg  )r   r  r  r  r   r  r  rX  rY  r  r  r  r  r  r   r   
valid_dimsr	  s   ffffffffffffff&  @r7   r  r    s   " 99;DL	LL
Q26V 
LL
Q26Q 
LLA'')a-o
 A!#::

1(:J+++AI;;*;;A!);R	
 	QY<<5::a=A-<<* A A	??j??UZZ]a-?~	
 
LL
a4++B!GtO	> 
LLq..\Q.	# 	#r9   c          ,          V ^8  d   QhR\         R\        R\        R\        R\        R\        R\        R\        R	\        R
\        R\        R\        R\        R\        R\        R\        R\        R\        R\        R\        R\        R\        /# )rA   r   r*  r'  r  r  r(  r  r   pTpHpW	dilationTrX  rY  r+  r,  r-  r.  r/  r0  rC  r#  )r   r   r  r   )rC   s   "r7   rD   rD     s     Y YYY 	Y 		Y
 	Y 	Y 	Y 	Y 	Y 	Y 	Y Y Y Y Y  !Y" #Y$ %Y& 'Y( )Y* +Y, -Yr9   c           
      0  a aaaaaaaaa	a
aaaaaaaaaaa S P                   p\        P                  ! S^ 8  ;'       d    S^ 8  ;'       d    S^ 8  VVV3R l4       \        P                  ! S^ 8  ;'       d    S^ 8  ;'       d    S^ 8  VVV3R l4       \        P                  ! S^ 8  ;'       d    S^ 8  ;'       d    S^ 8  VVV3R l4       \        P                  ! VR
9   VV 3R l4       \        V4       FB  oV^8X  d
   S^ 8X  d   K  \        P                  ! S P	                  S4      ^ 8  VVV 3R l4       KD  	  V'       d>   \        P                  ! SS8  ;'       d    SS8  ;'       d    SS8  VVVVVV3R l4       \        P                  ! S^,          S8  ;'       d!    S^,          S
8  ;'       d    S^,          S	8  VVVV	VV
3R l4       \        P                  ! S^8  ;'       d    S^8  ;'       d    S^8  VVVVVVV3R l4       R	# )r   c                     < R S RS  RS 2# )z5kernel size should be greater than zero, but got kT: z, kH: rR  r3   )r  r'  r  s   r7   rl   $pool3d_shape_check.<locals>.<lambda>  s    $fRDrd,r9   c                     < R S RS  RS 2# )z0stride should be greater than zero, but got dT: z, dH: rU  r3   )r  r(  r   s   r7   rl   rg    s    >rd&FSURVWr9   c                     < R S RS  RS 2# )z9dilation should be greater than zero, but got dilationT: z, dilationH: rW  r3   )rX  rd  rY  s   r7   rl   rg    s    #M)M)Vr9   c                  &   < S  R SP                    2# )z/: Expected 4D or 5D tensor for input, but got: r8  )rC  r   s   r7   rl   rg     s    7)J5;;-Xr9   c                  L   < S  R SP                    RSP                  S4       R2# )zZ: Expected input's non-batch dimensions to have positive length, but input has a shape of z and non-batch dimension z has length zero!)r   r   )rC  r   r   s   r7   rl   rg  )  s.    ) --2[[M+EJJqM?:KMr9   c                  .   < R S RS  RS RS RS RS R2# )zinput image (T: r  r  z ) smaller than kernel size (kT:  kH:  kW: r   r3   )r,  r+  r-  r  r'  r  s   r7   rl   rg  3  s4    "5'gYd6( C$$&4uRDbT<r9   c                  ,   < R S RS RS  RS RS RS 2# )zHpad should be smaller than or equal to half of kernel size, but got kT: rn  rm  z padT: z padW: z padH: r3   )r  r'  r  rb  ra  rc  s   r7   rl   rg  ;  s1    $eB4uRDt72$gbTKr9   c                  :   < R S RS RS  RS RS RS RS RS R2# r^  r3   )r,  r+  r-  r*  r/  r.  r0  s   r7   rl   rg  C  sD    !'!E7!G9AfX F((/y%'!F8 L'(r9   Nr%  )r   r]   ro   r   r   )r   r*  r'  r  r  r(  r  r   ra  rb  rc  rd  rX  rY  r+  r,  r-  r.  r/  r0  rC  r#  r   r   s   fffffffffffffffffffff& @r7   r&  r&    s   0 ::D	LL
Q$$26$$b1f	
 
LL
Q$$26$$b1f	
 
LLA99)a-99IM	
 
LLX
 4[19aJJqMA	
	  RK::GrM::fl 	
 
LL
Q"66a266"q&B,	
 	
 
LL
33v{33w!|	
 	
r9   c                    V P                   p\        V VVVVVVV	V
VVVVVVVVVVVV4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       R# )r  Nr   r&  r  )r   r  r   r*  r'  r  r  r(  r  r   ra  rb  rc  rd  rX  rY  r+  r,  r-  r.  r/  r0  rC  r   s   &&&&&&&&&&&&&&&&&&&&&&& r7   max_pool3d_backward_shape_checkrs  K  s    2 ::D








+0 ;dQh8;dQh6;dQh8;dQh77D$(G47D$(E27D$(G47D$(F3r9   c          &          V ^8  d   QhR\         R\         R\        R\        R\        R\        R\        R\        R	\        R
\        R\        R\        R\        R\        R\        R\        R\        R\        R\        /# )rA   r   r  r*  r'  r  r  r(  r  r   ra  rb  rc  r+  r,  r-  r.  r/  r0  rC  )r   r   r  )rC   s   "r7   rD   rD     s     38 383838 38 		38
 	38 	38 	38 	38 	38 	38 	38 	38 38 38 38  !38" #38$ %38& '38r9   c                    V P                   p\        V VVVVVVVV	V
V^^^VVVVVVVR4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       \        VVV^,
          V4       R# )r2   TNrr  )r   r  r*  r'  r  r  r(  r  r   ra  rb  rc  r+  r,  r-  r.  r/  r0  rC  r   s   &&&&&&&&&&&&&&&&&&& r7   r9  r9    s    * ::D








			-2 ;dQh8;dQh6;dQh8;dQh7r9   c                     R  pV! RV4      w  rx\         P                  ! \        V4      R
9   R 4       \        V4      ^ 8X  d   YxrMV! RV4      w  rV! RV4      w  rV! RV4      w  rV P                  R4      pV P                  R4      pV P                  R4      p\        P
                  ! V 4      pV\         P                  8X  d+   \         P                  ! V P                  4       ^8H  R 4       MWV\         P                  8X  d+   \         P                  ! V P                  4       R9   R 4       M\         P                  ! RR	 4       \        VW{WV4      p\        VWWV4      p\        V VVV	V
VVVVVVVVVV4       VVV3# )c                    a  \         P                  ! \        V4      R9   V 3R l4       V^ ,          p\        V4      ^8X  d   TMV^,          pW#3# )r2   c                     < R S  R2# )zmax_pool2d: r  r3   r  s   r7   rl   Emax_pool2d_checks_and_compute_shape.<locals>.unpack.<locals>.<lambda>  r  r9   r  r  r  s   f&  r7   r  3max_pool2d_checks_and_compute_shape.<locals>.unpack  r  r9   r_  c                      R # )zOmax_pool2d: stride must either be omitted, a single int, or a tuple of two intsr3   r3   r9   r7   rl   5max_pool2d_checks_and_compute_shape.<locals>.<lambda>  r  r9   r   r`  rD  c                      R # )zMnon-empty 4D (batch mode) tensor expected for input with channels_last layoutr3   r3   r9   r7   rl   r|    s    cr9   c                      R # )z9non-empty 3D or 4D (batch mode) tensor expected for inputr3   r3   r9   r7   rl   r|        Or9   Fc                      R # )zAUnsupported memory format. Supports only ChannelsLast, Contiguousr3   r3   r9   r7   rl   r|    s    Wr9   r  r  r  r   re  )r]   ro   r   r   rV   r   rg  r   r   r  r  )r   r_  r   r`  rD  r  r  r  r  r  r   r  r  rX  rY  r  r  r  r   r  r  s   &&&&&&               r7   r  r    sy    M;/FB	LLFy a 6{aB&)	7+JD!*h7I**R.K**R.KBJ//6M+++IIK1c	
 
%11	1IIK6!O	

 	W	

 (RriXL&z2RIVK



$ k11r9   c                   a aaaaa \        SW#WEV4      w  poo\        P                  ! SP                  S P                  8H  V V3R  l4       VoSP                  oVVVV3R lp	V	! S 4       V	! V4       \
        P                  ! S4      p
\        P                  ! SP                  SP                  SP                  V
R7      # )c                  <   < R SP                    RS P                    2# )zExpected dtype z  for `gradOutput` but got dtype r   rV  s   r7   rl   7meta_max_pool2d_with_indices_backward.<locals>.<lambda>!  s    /$**-MkN_N_M`ar9   c                    < \        V SS^,
          S4       \        V SS^,
          S4       \        V SS^,
          S4       R# )r  N)r  )r,  r	  r   r  r  s   &r7   _check_dim_size>meta_max_pool2d_with_indices_backward.<locals>._check_dim_size'  s9    q$q,7q$q,7q$q+6r9   r  )
r  r]   ro   re   r   rV   r   r   r   r   )r  r   r_  r   r`  rD  r  r   r  r  r   r	  r   r  r  s   ff&&&&&&   @@@@r7   %meta_max_pool2d_with_indices_backwardr    s      	,k7i		
 
LL

k'''a
 L99D7 7
 K G//5M;;

jj{{#	 r9   c                    \        WW#WE4      w  pppV P                  4       ^8X  d   V P                  R4      M^p	\        P                  ! V 4      p
V P                  4       ^8X  d   WgV.pMWWx.p\
        P                  ! VV P                  V P                  V
R7      \
        P                  ! V\
        P                  V P                  V
R7      3# r  )
r  r   r   rV   r   r]   r   re   r   r   r  s   &&&&&&      r7   meta_max_pool2d_with_indicesr  8  s     	,FX		
  %yy{a/UZZ^QF//6Myy{a;7\?++<<'		
 	++<<'		
 r9   c                   a aaa
aa \         P                  ! S P                  R9   V 3R l4       S P                  p\        V^,
          V4       F1  o
\         P                  ! S P	                  S
4      ^ 8  V
V 3R l4       K3  	  \         P                  ! \        S4      ^8H  R 4       \         P                  ! \        V4      ^8H  R 4       S P	                  R4      pS P	                  R4      oS P	                  R4      oV^8X  d   S P	                  ^ 4      pM^p\         P                  ! S P                  SP                  8H  R 4       \         P                  ! SP                  ^8H  V3R l4       SP	                  ^ 4      pSP	                  ^4      pSP	                  ^4      o
\         P                  ! Wv8  R 4       \         P                  ! W8H  R 4       \         P                  ! S
^8H  V
3R	 l4       \         P                  ! V^ ,          S^ ,          ,           ^,
          S8*  VV3R
 l4       \         P                  ! V^,          S^,          ,           ^,
          S8*  VV3R l4       S P                  4       ^8X  d   WeV^ ,          V^,          .p	MWR^ ,          V^,          .p	\         P                  ! V	S P                  S P                  R7      \         P                  ! V	\         P                  S P                  R7      3# )r  c                  "   < R S P                    2# )z:fractional_max_pool2d: Expected 3D or 4D tensor, but got: rv  r   s   r7   rl   ,meta_fractional_max_pool2d.<locals>.<lambda>c  s    LTYYKXr9   c                  2   < R SP                  4        RS  R2# )z_fractional_max_pool2d: Expected input to have non-zero  size for non-batch dimensions, but got rJ  z emptyr   )r   r   s   r7   rl   r  j  s%     77;yy{mCSTUSVV\^r9   c                      R # )zNfractional_max_pool2d: kernel_size musteither be a single int or tuple of Intsr3   r3   r9   r7   rl   r  q       2r9   c                      R # )zOfractional_max_pool2d: output_size must either be a single int or tuple of Intsr3   r3   r9   r7   rl   r  v  r  r9   c                      R # )z6Expect _random_samples to have the same dtype as inputr3   r3   r9   r7   rl   r    s    Hr9   c                  "   < R S P                    2# )z1Expect _random samples to have 3 dimensions got, rv  )random_sampless   r7   rl   r    s    CNDWDWCXYr9   c                      R # )z=Expect _random_samples.size(0) no less then input batch size.r3   r3   r9   r7   rl   r    r   r9   c                      R # )z<Expect _random_samples.size(1) equals to input channel size.r3   r3   r9   r7   rl   r        Nr9   c                     < R S  R2# )z/Expect _random_samples.size(2) equals to 2 got .r3   )r   s   r7   rl   r    s    #RSTRUUV!Wr9   c                  "   < R S^ ,           RS  2# )z%fractional_max_pool2d: kernel height z' is too large relative to input height r3   )input_heightr_  s   r7   rl   r    s    7A7GGno{n|}r9   c                  "   < R S^,           RS  2# )z$fractional_max_pool2d: kernel width z& is too large relative to input width r3   )input_widthr_  s   r7   rl   r    s    6{1~6FFlmxlyzr9   r  re  r  r  r   )r]   ro   r   r   r   r   re   r   r   r   r   )r   r_  rB  r  r   input_channelsinput_batchr  cr   r   r  r  s   ff&f      @@@r7   meta_fractional_max_pool2dr  _  sf   	LL		VX 99D4!8T"IIaL1^	
 # 
LLKA	2
 
LLKA	2 YYr]N99R=L))B-Kqyiil	LL

n***H 
LLq Y
 	AAAAAA	LL	O 
LL	N 
LLaWX	LLAQ'!+|;} 
LLAQ'!+{:z
 xxzQ[^[^LAA? 	**;;	

 	++;;	
 r9   c                    \         P                  ! \        V4      R	9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p\         P                  ! V'       * ;'       g    \        V4      R	9   R 4       V'       g   TMV^ ,          p	V'       g   TM\        V4      ^8X  d   T	MV^,          p
V'       g   TM\        V4      ^8X  d   T	MV^,          p\         P                  ! \        V4      R	9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p\         P                  ! \        V4      R	9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p\         P                  ! V P                  R
9   R 4       V P                  ^8X  d   V P	                  R4      M^pV P	                  R4      pV P	                  R4      pV P	                  R4      pV P	                  R4      p\        VWlWV4      p\        VW}V
VV4      p\        VWVVV4      p\        V VVVVV	V
VVVVVVVVVVVVVR4       V P                  ^8H  ;'       d(    \        P                  ! V 4      \         P                  8H  pV P                  ^8X  d   VVVV3pMVVVVV3pV P                  V4      pV P                  V\         P                  R7      pV'       dA   VP                  \         P                  R7      pVP                  \         P                  R7      pVV3# )r2   c                      R # zMmax_pool3d: kernel_size must either be a single int, or a tuple of three intsr3   r3   r9   r7   rl   .meta_max_pool3d_with_indices.<locals>.<lambda>      _r9   c                      R # zQmax_pool3d: stride must either be omitted, a single int, or a tuple of three intsr3   r3   r9   r7   rl   r        cr9   c                      R # zImax_pool3d: padding must either be a single int, or a tuple of three intsr3   r3   r9   r7   rl   r        [r9   c                      R # zJmax_pool3d: dilation must be either a single int, or a tuple of three intsr3   r3   r9   r7   rl   r    r  r9   c                      R # r   r3   r3   r9   r7   rl   r    r!  r9   zmax_pool3d_with_indices()r   r   r$  r%  r  r  r  r   )r]   ro   r   r   r   r  r&  rV   r   r  r   r   rr  )r   r_  r   r`  rD  r  r'  r  r  r(  r  r   ra  rb  rc  rd  rX  rY  ry  r*  r+  r,  r-  r.  r/  r0  rg  r   r   r   s   &&&&&&                        r7   meta_max_pool3d_with_indicesr    s    
LLKF"_ 
QB;1$+a.B;1$+a.B	LL
++c&kV+c vayBc&kQ&6F1IBc&kQ&6F1IB	LLG[ 
B7|q gajB7|q gajB	LLH\ I ]a/	Xa[I ]a/	Xa[I	LL

fK
  %zzQUZZ^AFjjnGJJrNEjjnGZZ^F yIE"7BB	9MG!&""iKF








#+4 	

aXXE77>%BXBXX  zzQeWf5	WeWf=	
//)
$Cooiu{{o;Gff5#9#9f:**5+A+A*B<r9   c                    \         P                  ! \        V4      R9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p	\        V4      ^8X  d   TMV^,          p
\         P                  ! V'       * ;'       g    \        V4      R9   R 4       V'       g   TMV^ ,          pV'       g   T	M\        V4      ^8X  d   TMV^,          pV'       g   T
M\        V4      ^8X  d   TMV^,          p\         P                  ! \        V4      R9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p\         P                  ! \        V4      R9   R 4       V^ ,          p\        V4      ^8X  d   TMV^,          p\        V4      ^8X  d   TMV^,          p\         P                  ! VP                  R	9   R 4       VP	                  R
4      pVP	                  R4      pVP	                  R4      pVP	                  R4      pV P	                  R4      pV P	                  R4      pV P	                  R4      p\        VV VVVV	V
VVVVVVVVVVVVVVVR4       VP                  ^8H  ;'       d(    \        P                  ! V4      \         P                  8H  pVP                  VP                  4      pV'       d!   VP                  \         P                  R7      pV# )r2   c                      R # r  r3   r3   r9   r7   rl   7meta_max_pool3d_with_indices_backward.<locals>.<lambda>&  r  r9   c                      R # r  r3   r3   r9   r7   rl   r  .  r  r9   c                      R # r  r3   r3   r9   r7   rl   r  6  r  r9   c                      R # r  r3   r3   r9   r7   rl   r  >  r  r9   c                      R # r   r3   r3   r9   r7   rl   r  F  r!  r9   z"max_pool3d_with_indices_backward()r   r$  r%  r  r  r  r   )r]   ro   r   r   r   rs  rV   r   r  r   r   rr  )r  r   r_  r   r`  rD  r  r   r'  r  r  r(  r  r   ra  rb  rc  rd  rX  rY  r*  r+  r,  r-  r.  r/  r0  rg  r  s   &&&&&&&&                     r7   %meta_max_pool3d_with_indices_backwardr    s    
LLKF"_ 
QB;1$+a.B;1$+a.B	LL
++c&kV+c vayBc&kQ&6F1IBc&kQ&6F1IB	LLG[ 
B7|q gajB7|q gajB	LLH\ I ]a/	Xa[I ]a/	Xa[I	LL

fK
 jjnGJJrNEjjnGZZ^FR Er"Gb!F#








,/8 	

aXXE77>%BXBXX  -J]]1G1G]H
r9   c                0    V ^8  d   QhR\         R\         /# )rA   r   gridr_  )rC   s   "r7   rD   rD   y  s     %
 %
V %
6 %
r9   c                   a aa \         P                  ! S P                  SP                  8H  VV 3R  l4       \         P                  ! S P                  \         P                  8H  ;'       d    SP                  \         P                  8H  VV 3R l4       \         P                  ! S P
                  ^ ,          SP
                  ^ ,          8H  VV 3R l4       \         P                  ! SP
                  R,          S P                  ^,
          8H  VV 3R l4       \        ^S P                  4       F3  o\         P                  ! S P
                  S,          ^ 8  VV 3R l4       K5  	  R# )c                  <   < R SP                    RS P                    2# )zNgrid_sampler(): expected input and grid to be on same device, but input is on z and grid is on rN  r  r   s   r7   rl   +check_grid_sampler_common.<locals>.<lambda>|  s"    \\N"24;;-Ar9   c                  <   < R SP                    RS P                    2# )zTgrid_sampler(): expected input and grid to have torch.strided layout, but input has z and grid has )r   r  s   r7   rl   r    s!    nT[[MCr9   c                  <   < R SP                    RS P                    2# )zZgrid_sampler(): expected grid and input to have same batch size, but got input with sizes  and grid with sizes r8  r  s   r7   rl   r    s"      %},A$**Or9   c                  J   < R SP                   ^,
           RS P                   2# )z+grid_sampler(): expected grid to have size z, in last dimension, but got grid with sizes )r   r   r  s   r7   rl   r    s'    9%**q.9I J226**?r9   c                  *   < R SP                    RS  R2# )zYgrid_sampler(): expected input to have non-empty spatial dimensions, but input has sizes rJ  rK  r8  ra  s   r7   rl   r    rb  r9   Nr   )r]   ro   r   r   r<  r   r   r   )r   r  r   s   ff@r7   check_grid_sampler_commonr  y  s    	LL#	
 
LL%FF$++*F	
 
LLA$**Q-'	
 
LL

2%**q.(	
 1ejj!KKNQ	
 "r9   c                   "    ] tR tRt^ t^t^tRtR# )GridSamplerInterpolationi  r3   N)r   
__module____qualname____firstlineno__BILINEARNEARESTBICUBIC__static_attributes__r3   r9   r7   r  r    s    HGGr9   r  c                <    V ^8  d   QhR\         R\         R\        /# )rA   r   r  interpolation_mode)r   r   )rC   s   "r7   rD   rD     s!       v 3 r9   c                 <  a a \         P                  ! S P                  ^8H  ;'       d    S P                  SP                  8H  VV 3R l4       \         P                  ! S P                  ^8H  ;'       d    V\        P                  P
                  8H  '       * R 4       R# )r  c                  <   < R SP                    RS P                    2# )zdgrid_sampler(): expected 5D input and grid with same number of dimensions, but got input with sizes r  r8  r  s   r7   rl   'check_grid_sampler_3d.<locals>.<lambda>  s!    449KK=#DJJ<1r9   c                      R # )z<grid_sampler(): bicubic interpolation only supports 4D inputr3   r3   r9   r7   rl   r    r  r9   N)r]   ro   r   r  r  r  )r   r  r  s   ff&r7   check_grid_sampler_3dr    sy    	LL

a33EJJ$))3	
 
LLJJ!O M M"&>&F&F&L&LL	
 	Or9   c                     V^ ,          pV'       d(   \         P                  ! V\         P                  R7      pMRp\         P                  ! V\         P                  R7      p	W3# r   r   N)r]   r  r   r   
r  r   r  r  padding_modealign_cornersr  input_requires_gradr  	grad_grids
   &&&&&&&   r7   grid_sampler_2d_backward_metar    sO     &a.%%e5;R;RS

  U5L5LMI""r9   c                    \        W4       \        WV4       V P                  ^ ,          pV P                  ^,          pVP                  ^,          pVP                  ^,          pVP                  ^,          p	V P                  WVWxV	34      # r  )r  r  r   r   )
r   r  r  r  r  r4  Cout_Dout_Hout_Ws
   &&&&&     r7   grid_sampler_3dr    sn     e*%'9:AAAAJJqMEJJqMEJJqME??A%677r9   r  c                     \        W4       \        WV4       V^ ,          pV'       d(   \        P                  ! V\        P                  R7      pMRp\        P
                  ! V\        P                  R7      p	W3# r  )r  r  r]   r  r  r   r  s
   &&&&&&&   r7   grid_sampler_3d_backwardr    sg     e*%'9:%a.%%!?!?

 
  U5S5STI  r9   c                     VP                  R 4      pV'       g   \        P                  ! V4      pWCR &   \        P                  ! V .VO5/ VB # r   )rd   rV   	get_dtyper]   r   )r   r  rX   r  re   s   &&*, r7   fullr    sA    JJwE
+7O;;t-d-f--r9   c           	      j   V\         P                  8X  d   \         P                  ! VR J R 4       \         P                  ! ^ Vf   V P                  MTTVf   V P
                  MTVR7      pV P                  '       d?   VP                  V P                  4       V P                  4       V P                  4       4       M/VP                  V P                  4       V P                  4       ^ 4       VP                  R4       V# \        P                  P                  V VVVVVR7      pVP!                  ^ 4       V# )Nc                      R # )z9memory format option is only supported by strided tensorsr3   r3   r9   r7   rl   zeros_like.<locals>.<lambda>  r  r9   r#  Tr   )r]   
sparse_cooro   r   re   r   	is_sparsesparse_resize_and_clear_r   
sparse_dim	dense_dimr   _coalesced_r-   r   defaultfill_)r   re   r   r   r   r   r(  s   &&&&&& r7   r  r    s     !!!T!O	

 kk %$**5"(.4;;f!
 >>>((		T__.0@ ((dhhj!D

//
!
!# " C IIaLJr9   r   c                    Vf   \         P                  ! 4       pVf   \         P                  ! 4       pVf   \         P                  p\         P                  ! WW#VR7      # Nr#  r]   r   get_default_devicer<  r   r   re   r   r   r   r   s   &$$$$$r7   	meta_onesr  /  P     }'')~))+~;;&J r9   c                    Vf   \         P                  ! 4       pVf   \         P                  ! 4       pVf   \         P                  p\         P                  ! WW#VR7      # r  r  r  s   &$$$$$r7   
meta_zerosr  E  r  r9   c                     \        V 4      # rH   _scatter_meta_output)r   ru  r   r   s   &&&&r7   meta_select_scatterr  [      %%r9   c                     \        V 4      # rH   r  )r   ru  r   r   r   steps   &&&&&&r7   meta_slice_scatterr  `  r   r9   c                     ^ RI Hp V! V 4      '       g,   \        P                  ! V 4      ^8X  d   V P	                  4       # \
        P                  ! V 4      # )r   rm  )ro  rn  r]   rp  r   rV   clone_preserve_strides)r   rn  s   & r7   r  r  e  sB    K
 !&&5+L+LT+RVW+Wzz|''--r9   c                <    V ^8  d   QhR\         R\         R\        /# )rA   r   dim_post_exprwrap_scalar)r   r   )rC   s   "r7   rD   rD   q  s!       C d r9   c           	          V^ 8:  d   V'       g   \        RV R24      h^pV) pV^,
          pW8  g   W8  d   \        RV  RV RV R24      hV ^ 8  d	   W,          p V # )r   zdim_post_expr=z <= 0 but wrap_scalar is Falsezdim z out of bounds (r   r   r  )r   r  r  r  r  s   &&&  r7   r   r   q  s}      /MN  .C
!
C
yCItC5(8RuAFGG
QwJr9   c                 V    V P                  4       ^ 8X  d   ^# V P                  V,          # r  r  )r,  r   s   &&r7   ensure_nonempty_sizer    s!    11.!''#,.r9   c                 J  a aaa \        S P                  4       ^4      p\        SP                  4       ^4      p\        P                  ! W48H  R 4       \	        V4       FA  oSS8w  g   K  \        P                  ! \        SS4      \        S S4      8*  VVVV 3R l4       KC  	  R# )r2   c                      R # )zDIndex tensor must have the same number of dimensions as input tensorr3   r3   r9   r7   rl   $gather_shape_check.<locals>.<lambda>  s    Vr9   c                  V   < R S RSP                    2RSP                    RS  2,           # )z!Size does not match at dimension z expected index  to be no larger than self  apart from dimension r8  )r   r   r   r   s   r7   rl   r    s5    ;A3>Nu{{m\/

|;QRUQVWXr9   N)r  r   r]   ro   r   r  )r   r   r   	self_dims
index_dimsr   s   fff  @r7   gather_shape_checkr    s}    DHHJ"IUYY[!$J	LLV 98LL$UA.2FtQ2OOX r9   c                   a ^ RI Hp \        WP                  4       4      pV! SP	                  4       ^ 8H  4      pV'       gh   \
        P                  ! SP                  \
        P                  8H  ;'       g    SP                  \
        P                  8H  V3R l4       \        WS4       V P                  SP                  4      # )r   r|  c                  "   < R S P                    2# )z8gather(): Expected dtype int32/int64 for index, but got r   r   s   r7   rl   meta_gather.<locals>.<lambda>  s    Nu{{m\r9   )ro  r}  r   r   r   r]   ro   re   r   r   r  r   r   )r   r   r   sparse_gradr}  wrapped_dimis_index_emptys   &&f&   r7   meta_gatherr    s    D hhj1K#EKKMQ$67NKK5::%AA		)A\	
 	4e4>>%++&&r9   c                     V'       dH   V R 8X  d   R# V R8X  d   R# V R8X  d   R# V R8X  d   R# V R8X  d   R	# \         P                  ! R
R 4       R# V R8X  d   R# V R8X  d   R# \         P                  ! R
R 4       R# )r  
REDUCE_ADDr  REDUCE_MULTIPLYr`  REDUCE_MEANamaxREDUCE_MAXIMUMaminREDUCE_MINIMUMFc                      R # )z=reduce argument must be either sum, prod, mean, amax or amin.r3   r3   r9   r7   rl   #get_operator_enum.<locals>.<lambda>  s    Sr9   Naddmultiplyc                      R # )z/reduce argument must be either add or multiply.r3   r3   r9   r7   rl   r%    s    $Ur9   r  )reduce_use_new_optionss   &&r7   get_operator_enumr+    s{    e$ ##S	
 	e
"$UUVr9   c                 x  a  ^ RI Hp V! VP                  4       ^ 8g  4      '       d\   \        P                  ! VP
                  \        P                  8H  ;'       g    VP
                  \        P                  8H  V 3R l4       Ve5   \        P                  ! VP
                  VP
                  8H  V 3R l4       R# R# )r   )r  c                     < S  R 2# )z((): Expected dtype int32/int64 for indexr3   method_names   r7   rl   ,scatter_gather_dtype_check.<locals>.<lambda>  s    {m#KLr9   Nc                     < S  R 2# )z0(): Expected self.dtype to be equal to src.dtyper3   r.  s   r7   rl   r0    s    {m#STr9   )ro  r  r   r]   ro   re   r   r   )r/  r   r   src_optr  s   f&&& r7   scatter_gather_dtype_checkr3    s|    CU[[]a'((KK5::%AA		)AL	

 JJ'--'T	
 r9   c                     \        V ^4      # r1   )r  r  s   &r7   ensure_nonempty_dimr5    s    sA;r9   c                   a aaa ^ RI Hp V! SP                  4       ^ 8H  4      '       d   R# \        P                  ! \        S P                  4       4      \        SP                  4       4      8H  R 4       \        S P                  4       4      p\        V4       FC  pVS8X  d   K  \        SV4      p\        S V4      p\        P                  ! Wx8*  VVV 3R l4       KE  	  Se   \        P                  ! \        S P                  4       4      \        SP                  4       4      8H  R 4       \        V4       F9  p\        SV4      p\        SV4      p	\        P                  ! Wy8*  VV3R l4       K;  	  R# R# )r   r|  Nc                      R # )zCIndex tensor must have the same number of dimensions as self tensorr3   r3   r9   r7   rl   %scatter_shape_check.<locals>.<lambda>  rd  r9   c                  P   < R SP                    RSP                    2RS  2,           # )Expected index r  r  r8  )r   r   r   s   r7   rl   r8    s*    oekk]2Mdjj\Z&se,-r9   c                      R # )zBIndex tensor must have the same number of dimensions as src tensorr3   r3   r9   r7   rl   r8    s    Xr9   c                  <   < R S P                    RSP                    2# )r:  z to be no larger than src r8  )r   r2  s   r7   rl   r8    s    /%++6PQXQ^Q^P_`r9   )	ro  r}  r   r]   ro   r5  r   r   r  )
r   r   r   r2  r}  r  r   index_d_sizeself_d_size
src_d_sizes
   ffff      r7   scatter_shape_checkr@    s"   Dekkmq())	LLDHHJ'+>uyy{+KKU
 $DHHJ/I 98+E15*43'-	
  
+/B7;;=/QQX	
 y!A/q9L-gq9JLL*` " r9   c                     \        WP                  4       4      p\        R WV4       \        WW#4       Ve   \	        WE4       R# R# )scatterN)r   r   r3  r@  r+  )r   r   r   ru  r)  r*  r  s   &&&&&& r7   scatter_meta_implrC    s;     hhj1Ky$s;56'3 r9   c                 R    \        WW#R 4       V P                  V P                  4      # r&  rC  r   r   r   r   r   ru  s   &&&&r7   meta_scatter_addrH    s!    dU3>>$**%%r9   c                      \        WW#R 4       V # rE  rC  rG  s   &&&&r7   meta_scatter_add_rK    s    dU3Kr9   c                     \        V\        P                  4      '       d   TMR p\        WW%V4       V P	                  V P
                  4      # rH   )ry   r]   r   rC  r   r   r   r   r   src_or_valuer  ru  s   &&&&& r7   meta_scatterrO    s:     %\5<<@@,dCdV4>>$**%%r9   c                 h    \        V\        P                  4      '       d   TMR p\        WW%V4       V # rH   )ry   r]   r   rC  rM  s   &&&&& r7   meta_scatter_rQ  #  s,     %\5<<@@,dCdV4Kr9   c                z    V ^8  d   QhR\         R\         R\         R\        R\        R\        R\        R,          /# )	rA   queryr   r  	dropout_p	is_causalreturn_debug_maskr!  Nr   ra   r   )rC   s   "r7   rD   rD   2  sR     < <<	< < 	<
 < < 4<<r9   c           	      t   V P                  ^ 4      pV P                  ^4      pV P                  ^4      p	V P                  ^4      p
VP                  ^4      p\        P                  ! V 4      p\        P                  ! WxV	3\        P                  V P
                  R7      pV'       dk   V
^@8  d   ^MRp\        P                  ! W,          4      pV^8:  d   ^pM	VR8:  d   Rp\        P                  ! WxW3V P                  V P
                  R7      pM-\        P                  ! ^ V P                  V P
                  R7      p\        P                  P                  '       d%   \        P                  P                  4       '       g   \        V 4      R8X  dP   \        P                  ! R\        P                  RR7      p\        P                  ! R\        P                  RR7      pMN\        P                  ! ^\        P                  RR7      p\        P                  ! R\        P                  RR7      pVVRRV	VVVV3	# )r   r     r  r   Nr3   )r   r]   r   r   ra   r   r  ceilre   r]  r^  r   r  r   r   r>  )rS  r   r  rT  rU  rV  r!  r   	num_headsmax_seqlen_batch_qhead_dimmax_seqlen_batch_k	attention	logsumexpblocksize_cmax_seqlen_k
debug_maskseedoffsets   &&&&&&&            r7   (meta__scaled_dot_product_flash_attentionrf  1  s    AJ

1IAzz!}H!  'I	 23kk||I %]cyy!3!AB$L3&L[[$6E++<<

 [[%++ellK
 }}UZZ4466+e:LPU:U{{2UZZ?Ruzz&A{{Aell6BRu||FC 	
 
r9   c                    V ^8  d   QhR\         R\         R\         R\         R,          R\         R,          R\         R,          R\        R	\        R
\        R\        R,          /
# )rA   rS  r   r  	q_descaleN	k_descale	v_descalerT  rU  rV  r!  rW  )rC   s   "r7   rD   rD   r  s|      	  }	
 } }    4<r9   c
           	          V P                   \        P                  8X  d    V P                  \        P                  4      p \        V VVVVVV	4      # rH   )re   r]   r  rr  r  rf  )
rS  r   r  rh  ri  rj  rT  rU  rV  r!  s
   &&&&&&&&&&r7   2meta__scaled_dot_product_flash_attention_quantizedrl  q  sJ     {{e)))(3 r9   c                J    V ^8  d   QhR\         R\        \        R3,          /# )rA   rS  	res_shape.)r   rn   r   )rC   s   "r7   rD   rD     s$      S#Xr9   c                   a  \        S P                  4      V8X  d   \        P                  ! S 4      pV# \	        . ROV 3R lRR7      pV Uu. uF  qAV,          NK  	  pp\        \        V4      4       Uu. uF  qcP                  V4      NK  	  pp\        P                  ! VS P                  S P                  R7      P                  V4      pV# u upi u upi )r   c                 2   < SP                  4       V ,          # rH   r  )idxrS  s   &r7   rl   ,alloc_with_matching_layout.<locals>.<lambda>  s    %,,.*=r9   Tr   r  )r   r2   rA   r  )rn   r   r]   r   r   r   r   r   r   re   r   r   )rS  rn  r(  	dim_orderrq  permuted_shaper   final_permutes   f&      r7   alloc_with_matching_layoutrv    s     U[[Y&u% J =t
	 5>>ISC..I>5:3y>5JK5J+5JKkk%++ell

'-
  	 J ?Ks   C5Cc                    V ^8  d   QhR\         R\         R\         R\         R,          R\        R\        R\        R	\        R
\        R,          /	# )rA   rS  r   r  	attn_biasNcompute_log_sumexprT  rU  rV  r!  r   r   ra   )rC   s   "r7   rD   rD     sj     ( ((	( ( }	(
 ( ( ( ( 4<(r9   c	           	         V P                  ^ 4      p	V P                  ^4      p
V P                  ^4      pVP                  ^4      pVP                  R4      pWW3p\        W4      p\        P                  ! WV^3\        P                  V P
                  R7      p\        P                  ! R\        P                  RR7      p\        P                  ! R\        P                  RR7      pVVRRVVVVR3	# )r   r  r   Nr   r3   )r   rv  r]   r   ra   r   r   )rS  r   r  rx  ry  rT  rU  rV  r!  r;  r  S_QS_KVD_Vrn  r(  
logsum_exprd  re  s   &&&&&&&&&          r7   (meta__scaled_dot_product_cudnn_attentionr    s     	

1A

1A
**Q-C88A;D
**R.Cs I
$U
6C	
sAkk||J ;;rF;D[[5::f=F 	
 
r9   c                    V ^8  d   QhR\         R\         R\         R\         R,          R\        R\        R\        R	\        R,          /# )
rA   rS  r   r  rx  NrT  rU  rV  r!  rW  )rC   s   "r7   rD   rD     s`     0 00	0 0 }	0
 0 0 0 4<0r9   c           	         ^ ^ ^ rpV P                  4       ^8X  d   V P                  4       w  rrM6V P                  4       ^8X  d   V P                  4       w  rp^pM\        R4      hVP                  R4      pVP                  R4      p\        V P                  4      pWR&   \        V \        V4      4      p\        P                  ! WV
3\        P                  V P                  R7      p\        P                  ! R\        P                  RR7      p\        P                  ! R\        P                  RR7      pVVRRV
VVVR3	# )r   zquery must be 3D or 4Dr  r   Nr  r   r3   )r   r   rq  r   r   rv  rn   r]   r   ra   r   r   )rS  r   r  rx  rT  rU  rV  r!  r;  H_Qr|  rY   r}  r~  r   r(  r  rd  re  s   &&&&&&&&           r7   5meta__scaled_dot_product_fused_attention_overrideabler    s    QCAyy{aQ		jjl!34488B<D
**R.C U[[!IbM
$UE),<
=C	
kk||J ;;rF;D[[5::f=F 	
 
r9   r!  c          #         V ^8  d   QhR\         R\         R\         R\         R\         R\        \        ,          R\         R\         R	\         R
\         R\        R\        R\        R\        R\         R\         R\        R,          /# )rA   rL  rS  r   r  rx  grad_input_maskr   r`  	cum_seq_q	cum_seq_kmax_qmax_krT  rU  philox_seedphilox_offsetr!  N)r   r   r   r   ra   )rC   s   "r7   rD   rD      s     2 222 
2 	2
 2 $Z2 
2 2 2 2 2 2 2 2 2  !2$ 4<%2r9   c                    \         P                  ! V4      p\         P                  ! V4      p\         P                  ! V4      pVe   \         P                  ! V4      MR pVVVV3# rH   r  )rL  rS  r   r  rx  r  r   r`  r  r  r  r  rT  rU  r  r  r!  grad_qgrad_kr  grad_attn_biass   &&&&&&&&&&&&&&&&$    r7   >meta__scaled_dot_product_fused_attention_overrideable_backwardr    sZ    * e$Fc"Fe$F4=4IU%%i0tN66>11r9   c                    V ^8  d   QhR\         R\         R\         R\         R\         R\         R\         R\         R	\        R
\        R\        R\        R\         R\         R\        R,          /# )rA   rL  rS  r   r  r   r`  r  r  r  r  rT  rU  r  r  r!  Nr   r   ra   r   )rC   s   "r7   rD   rD      s     " """ 
" 	"
 
" " " " " " " " " " 4<"r9   c                     \         P                  ! V4      p\         P                  ! V4      p\         P                  ! V4      pVVV3# rH   r  )rL  rS  r   r  r   r`  r  r  r  r  rT  rU  r  r  r!  r  r  r  s   &&&&&&&&&&&&&&&   r7   'meta__scaled_dot_product_flash_backwardr    sA    , e$Fc"Fe$F66!!r9   c                    V ^8  d   QhR\         R\         R\         R\        R\        R\         R,          R\        R,          /# )	rA   rS  r   r  rT  rU  	attn_maskNr!  rW  )rC   s   "r7   rD   rD   <  sV      	  	
  } 4<r9   c                 $   V P                  ^ 4      pV P                  ^4      pV P                  ^4      p	\        P                  ! V 4      p
\        P                  ! VV	V3\        P                  V P
                  R7      P                  ^^4      pV
V3# )r   r  )r   r]   r   r   ra   r   rk  )rS  r   r  rT  rU  r  r!  r   r[  r\  r_  r`  s   &&&&&&&     r7   0meta__scaled_dot_product_flash_attention_for_cpur  7  s     AJ

1IA  'I	

 kk|| i1o  	 r9   c                    V ^8  d   QhR\         R\         R\         R\         R\         R\         R\        R\        R	\         R
,          R\        R
,          /
# )rA   rL  rS  r   r  r   r`  rT  rU  r  Nr!  rW  )rC   s   "r7   rD   rD   ^  st     !" !"!"!" 
!" 	!"
 
!" !" !" !" }!" 4<!"r9   c
                 r   \         P                  ! VP                  4       RVP                  VP                  R7      p
\         P                  ! VP                  4       RVP                  VP                  R7      p\         P                  ! VP                  4       RVP                  VP                  R7      pWV3# )r   r  r   rA   r2   r  )r]   empty_permutedr   re   r   )rL  rS  r   r  r   r`  rT  rU  r  r!  r  r  r  s   &&&&&&&&&&   r7   9meta__scaled_dot_product_flash_attention_for_cpu_backwardr  Y  s    & !!

kk||	F !!
iizz	F !!

kk||	F 6!!r9   c                    V ^8  d   QhR\         R\         R\         R\         R,          R\        R\        R\         R,          R	\        R,          R
\        R\        \         \         3,          /
# )rA   rS  r   r  r  NrT  rU  dropout_maskr!  
enable_gqarB   )r   ra   r   rn   )rC   s   "r7   rD   rD     s     - --	- - }	-
 - - 4-- 4<- - 66>-r9   c	                    a aaaaaaa R  p	V	! S 4      w  ooV	! V4      w  rV	! V4      w  rSP                   w  ooopVP                   w   pooVVVVVV VV3R lpV! 4       # )c                    V P                  4       ^8X  d   V P                  ^ 4      R3# V P                  4       ^8  d   ^p\        V P                  4       ^,
          4       F  pWP                  V,          ,          pK  	  V P	                  WP                  R4      V P                  R4      V P                  R4      4      R3# V R3# )r  TFr  r  r   )r   r   r   r   viewr   )r<   r   r   s   &  r7   	ensure_4dBmeta__scaled_dot_product_attention_math_for_mps.<locals>.ensure_4d  s    557a<;;q>4''UUWq[J1557Q;'ggaj(
 (66*ffRj!&&*affRjI4OOe8Or9   c                  
  < SP                  SSSS34      p SP                  SSSS34      pS
'       d   S	P                  4       ^8X  d&   V P                  ^ 4      p VP                  ^ 4      pW3# \        S	P                  RR 4      \        V P                  R,          4      ,           p\        S	P                  RR 4      \        VP                  R,          4      ,           pV P                  V4      p VP                  V4      pW3# )r  N:r2   r  Nr  )r   r   squeezer   r   r  )r   attnr   
attn_shaper   max_seq_lengthnum_headq_q_sizerS  
unsqueezedvalue_head_sizes       r7   sdpa_general_mpsImeta__scaled_dot_product_attention_math_for_mps.<locals>.sdpa_general_mps  s    llJ&/JK||Z6>JKyy{akk!n||A y	 !Sb!12T#))C.5II	!%++cr"23d4::c?6KK
hhy)yy,yr9   r8  )rS  r   r  r  rT  rU  r  r!  r  r  k_rY   v_r  r   r  r  r  r  r  r  s   f&&&&&&&&     @@@@@@@r7   /meta__scaled_dot_product_attention_math_for_mpsr    sh    	 u%NB
cNEBeEB&(hh#J&!,.HH)Aq./   r9   c                    V ^8  d   QhR\         R\         R\         R\         R,          R\        R\        R\        R,          /# )	rA   rS  r   r  rx  Nry  rU  r!  rz  )rC   s   "r7   rD   rD     sV     )) ))))	)) )) }	))
 )) )) 4<))r9   c           	         V P                  ^^4      p VP                  ^^4      pVP                  ^^4      pV P                  ^ 4      pV P                  ^4      p	V P                  R4      p
VP                  R4      p\        P                  ! WWV P                  V P
                  R7      p\        P                  P                  '       d3   \        P                  P                  4       '       d    V'       d   T	M^ pM.V'       d%   \        P                  ! V	^ ,          4      ^ ,          M^ p\        P                  ! WV3\        P                  V P
                  R7      pVP                  ^^4      p\        P                  ! R\        P                  RR7      p\        P                  ! R\        P                  RR7      pWVV3# )r2   r  r   r  r   r3   )rk  r   r]   r   re   r   r]  r^  r   r  r  rZ  ra   r   )rS  r   r  rx  ry  rT  rU  r!  r;  r4  r[  Kvr(  logsumexp_dimr  rd  re  s   &&&&&&&&         r7   ,meta__scaled_dot_product_efficient_attentionr    sB    OOAq!E
--1
COOAq!E

1A

1A

2I	BB
++aIU\\
RC}}UZZ4466	 0Q2D		!b&)B.!	
}%kk||J --1
C ;;rF;D[[5::f=FD&((r9   c                    V ^8  d   QhR\         R\         R\         R\         R\         R,          R\         R\         R	\         R
\         R\        R\        \        ,          R\        R\        R,          /# )rA   rL  rS  r   r  rx  Nr   r`  r  r  rT  r  rU  r!  )r   ra   r   r   )rC   s   "r7   rD   rD     s     4- 4-4-4- 
4- 	4-
 }4- 
4- 4- 4- 4- 4- $Z4- 4- 4<4-r9   c                 J   VP                  ^ 4      pVP                  ^4      pVP                  ^4      pVP                  ^4      pVP                  ^4      pVP                  ^4      p\        P                  ! WVV3RVP                  VP                  R7      p\        P                  ! WVV3RVP                  VP                  R7      p\        P                  ! WVV3RVP                  VP                  R7      pRpVe   V
^,          '       d   VP                  R4      pV^,          ^ 8X  d   TMV^,           V^,          ,
          p\        VP                  4       4      pVVR&   \        P                  ! VVP                  VP                  R7      pVRRV13,          pVVVV3# )r   r  N.r  r   )r   r]   r  re   r   r   r   )rL  rS  r   r  rx  r   r`  r  r  rT  r  rU  r!  r   r[  r  r]  
head_dim_vr  r  r  r  	grad_biaslastDimlastDimAligned	new_sizess   &&&&&&&&&&&&&             r7   +meta__scaled_dot_product_efficient_backwardr    sx   ( AJ

1IJJqMEzz!}HAJHHQKE!!	x0kk||	F !!	x0iizz	F !!	z2kk||	F I!3!3..$$+bLA$57R<'TV,;V)*	&	"KKY__Y5E5E
	 c8G8m,	669,,r9   c          !          V ^8  d   QhR\         R\         R\         R\         R\         R\         R\         R\         R	\         R
\         R\         R\        R\        R\        R\        R\        R,          /# )rA   rL  rS  r   r  r   r`  r  r  rx  r  r  r  r  rT  rU  r!  Nr  )rC   s   "r7   rD   rD   !  s     " """ 
" 	"
 
" " " " " " " " " " "  4<!"r9   c                     \         P                  ! V4      p\         P                  ! V4      p\         P                  ! V4      pVVV3# rH   r  )rL  rS  r   r  r   r`  r  r  rx  r  r  r  r  rT  rU  r!  r  r  r  s   &&&&&&&&&&&&&&&&   r7   'meta__scaled_dot_product_cudnn_backwardr    sA    . e$Fc"Fe$F66!!r9   c          #      b   V ^8  d   QhR\         R\         R\         R\         R,          R\         R,          R\        R\        R	\        R
\        R\        R\        R,          R\        R,          R\        R,          R\         R,          R\         R,          R\         R,          R\        R,          /# )rA   rS  r   r  r  Nr  r  r  rT  rU  rV  r!  window_size_leftwindow_size_right	seqused_kalibi_slopesblock_tabler=  r  )rC   s   "r7   rD   rD   >  s     J JJ	J J }	J
 }J J J J J J 4<J DjJ TzJ }J 4-J  $!J" d
#Jr9   c                 <   Vf   V P                  ^ 4      MVP                  4       ^,
          pVf   V P                  ^4      MTpVf   VP                  ^4      MTpV P                  R4      pV P                  R4      p\        P                  ! V 4      pVf6   \        P                  ! VVV3\        P
                  V P                  R7      pMDV P                  ^ 4      p\        P                  ! VV3\        P
                  V P                  R7      pV	'       dn   V^@8  d   ^MRp\        P                  ! VV,          4      pV^8:  d   ^pM	VR8:  d   Rp\        P                  ! VVVV3V P                  V P                  R7      pM-\        P                  ! ^ V P                  V P                  R7      pR R pp\        P                  P                  '       dt   \        P                  P                  4       '       dP   \        P                  ! R\        P                  RR7      p\        P                  ! R\        P                  RR7      pMN\        P                  ! ^\        P                  RR7      p\        P                  ! R\        P                  RR7      pVVVVV3# )Nr  rY  r   r  r   r3   )r   r   r]   r   r   ra   r   r  rZ  re   r]  r^  r   r  r   r>  )rS  r   r  r  r  r  r  rT  rU  rV  r!  r  r  r  r  r  r=  r   r\  r^  r[  r]  r_  r`  total_qra  rb  rc  rd  re  s   &&&&&&&&&&&&&&&&&             r7   meta__flash_attention_forwardr  9  s   8 #,"3A9JQ9NJ*3*;A(1(9!u

2Izz"~H   'IKK$67++<<
	 **Q-KK ELL
	 %]cyy!3k!AB$L3&L[[$6E++<<

 [[%++ellK
 &D}}UZZ4466{{2UZZ?Ruzz&A{{Aell6BRu||FC r9   c          %      n   V ^8  d   QhR\         R\         R\         R\         R\         R,          R\         R,          R\        R	\        R
\        R\        R\        R\        R,          R\        R,          R\        R,          R\         R,          R\         R,          R\         R,          R\        R,          /# )rA   r   rS  r   r  r  Nr  r  r  rT  rU  rV  r!  r  r  r  r  r  r=  r  )rC   s   "r7   rD   rD     s     & &	&& 
& 	&
 }& }& & & & & & 4<& Dj& Tz& }&  4-!&" $#&$ d
%&r9   c                 F    \        VVVVVVVVV	V
VVVVVV4      w  pp  pV# rH   )r  )r   rS  r   r  r  r  r  r  rT  rU  rV  r!  r  r  r  r  r  r=  rY   r`  s   &&&&&&&&&&&&&&&&&&  r7   0meta__flash_attention_forward_no_dropout_inplacer    sQ    * :!Ay!Q$ r9   c          %      |   V ^8  d   QhR\         R\         R\         R\         R,          R\         R,          R\        R\        R	\        R
\        R\        R\         R,          R\         R,          R\         R,          R\        R,          R\        R,          R\        R,          R\         R,          R\         R,          /# )rA   rS  r   r  r  Nr  r  r  rT  rU  rV  rh  ri  rj  r!  r  r  r  r  r  )rC   s   "r7   rD   rD     s     ' ''	' ' }	'
 }' ' ' ' ' ' }' }' }' 4<' Dj'  Tz!'" }#'$ 4-%'r9   c                     V P                   \        P                  8X  d    V P                  \        P                  4      p \        V VVVVVVVVV	VVVVV4      # rH   )re   r]   r  rr  r  r  )rS  r   r  r  r  r  r  rT  rU  rV  rh  ri  rj  r!  r  r  r  r  s   &&&&&&&&&&&&&&&&&&r7   'meta__flash_attention_forward_quantizedr    sb    * {{e)))(( r9   c          #         V ^8  d   QhR\         R\         R\         R\         R\         R\         R\         R\         R	\        R
\        R\        R\        R\         R\         R\        R,          R\        R,          R\        R,          /# )rA   rL  rS  r   r  r   r`  r  r  r  r  rT  rU  r  r  r!  Nr  r  r  )rC   s   "r7   rD   rD     s     , ,,, 
, 	,
 
, , , , , , , , , , 4<,  Dj!," Tz#,r9   c                     \         P                  ! V4      p\         P                  ! V4      p\         P                  ! V4      pVVV3# rH   r  )rL  rS  r   r  r   r`  r  r  r  r  rT  rU  r  r  r!  r  r  
grad_querygrad_key
grad_values   &&&&&&&&&&&&&&&&&   r7   meta__flash_attention_backwardr    sA    0 !!%(J$H!!%(Jx++r9   c                J   V ^8  d   QhR\         R\         R\         R\         R,          R\         R,          R\         R,          R\        R,          R	\        R,          R
\        R\        R\        R\        R,          R\         R,          R\         R,          R\        R,          /# )rA   rS  r   r  r  Ncu_seqlens_qcu_seqlens_kmax_seqlen_qrb  rT  custom_mask_typery  r!  causal_diagonalseqlen_kwindow_sizer  )rC   s   "r7   rD   rD     s     /S /S/S	/S /S 4-	/S
 4-/S 4-/S */S */S /S /S /S 4</S d]/S tm/S t/Sr9   c           	         V P                  ^ 4      pV P                  ^4      pVP                  ^4      pV P                  R4      pVP                  R4      p\        P                  ! VVVVV P                  V P                  R7      pVe   VP                  ^ 4      ^,
          MTpTpVe   Vf   \        R4      hTpVe   TMTpV
'       d%   \        P                  ! V^ ,          4      ^ ,          M^ p\        P                  ! VVV3\        P                  V P                  R7      p\        P                  ! R\        P                  RR7      p\        P                  ! R\        P                  RR7      pVVVVVV3# )r   r  z;max_seqlen_q must not be None when cu_seqlens_q is providedr   r  r   r3   )
r   r]   r   re   r   r   r  rZ  ra   r   )rS  r   r  r  r  r  r  rb  rT  r  ry  r!  r  r  r  r;  r4  r4  r[  r  r(  logsumexp_batch_dimactual_max_seqlen_qactual_max_seqlen_kr  r  rd  re  s   &&&&&&&&&&&&&&&             r7   !meta__efficient_attention_forwardr    sE   , 	

1A

1AA

2I	BB
++aIrU\\
RC7C7O,++A.2VW M  +*6*B,4F		%*+b0A  	i7kk||J ;;rF;D[[5::f=F
D&*=?RRRr9   c          $      ^   V ^8  d   QhR\         R\         R\         R\         R\         R,          R\         R,          R\         R,          R	\        P                  R
\        P                  R\         R\        R\         R\         R\        R\
        R\        R,          R\        R,          R\
        /# )rA   rL  rS  r   r  r  Nr  r  r  rb  r`  rT  r  r  r  bias_requires_gradr!  num_splits_keyshared_storage_dqdkdv)r   r]   SymIntra   r   r   )rC   s   "r7   rD   rD   ;  s     47 474747 
47 	47
 4-47 4-47 4-47 ,,47 ,,47 47 47 47 47 47 47  4<!47" $J#47$  %47r9   c                 >   V'       Ed   \         P                  ! VP                  ^,          VP                  ^,          8H  R 4       \         P                  ! VP                  ^,          VP                  ^,          8H  R 4       \         P                  ! . VP                  ^ R O^NVP                  R,          NVP                  R,          N5VP                  VP
                  R7      pVP                  R	^ 4      pVP                  R	^4      pVP                  R	^4      pMB\         P                  ! V4      p\         P                  ! V4      p\         P                  ! V4      pVe   VP                  R4      pV^,          ^ 8X  d   TMV^,           V^,          ,
          p\        VP                  4       4      pVVR&   \         P                  ! VVP                  VP
                  R7      pVRRV13,          pM"\         P                  ! R
VP
                  R7      pVVVV3# )r2   c                      R # )z,seqlen must match for `shared_storage_dqdkdvr3   r3   r9   r7   rl   4meta__efficient_attention_backward.<locals>.<lambda>R  s    Br9   c                      R # )z3embedding dim must match for `shared_storage_dqdkdvr3   r3   r9   r7   rl   r  V      Ir9   r  N.rN  r  r   r  r3   )
r]   ro   r   r   re   r   r  r   r   r   )rL  rS  r   r  r  r  r  r  rb  r`  rT  r  r  r  r  r!  r  r  chunkr  r  r  r  r  r  r  s   &&&&&&&&&&&&&&&&&&        r7   "meta__efficient_attention_backwardr  6  s   2 KKNciil*B	
 	KKNciil*I	
 Eekk!BEEEKKOEU[[_E++<<

 \\"a(
<<A&\\"a(
%%e,
##C(%%e,
))B-$+bLA$57R<'TV,;V%	&	"KK	DKKP	c8G8m,	KK5<<8	xY66r9   c                .   V ^8  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R	\        /# 
rA   r   r  scale_ascale_br  Nscale_resultr  use_fast_accumr]   r   re   r   )rC   s   "r7   rD   rD   r  s     AY AY
,,AY
,,AY \\AY \\	AY
 ,,
AY ,,%AY {{T!AY AYr9   c                   a aaaaaaaa R  p\         P                  ! S P                  4       ^8H  ;'       d    SP                  4       ^8H  VV 3R l4       \         P                  ! V! S P                  4      ;'       d    V! SP                  4      VV 3R l4       \	        S 4      R8X  g"   \	        S 4      R8X  g   \	        S 4      R8X  Ed?   R p	R p
R p\	        S 4      R8w  Ed
   \         P                  ! V	! S P                  4       4      ;'       g	    V! S 4      V 3R	 l4       \         P                  ! V
! SP                  4       4      ;'       g	    V! S4      V3R
 l4       \         P                  ! S P                  ^4      ^,          ^ 8H  V 3R l4       \         P                  ! SP                  ^ 4      ^,          ^ 8H  ;'       d    SP                  ^4      ^,          ^ 8H  V3R l4       S P                  w  ooSP                  ^4      oSP                  \         P                  8H  ;'       d&    SP                  \         P                  8H  ;'       gC    SP                  \         P                  8H  ;'       d    SP                  \         P                  8H  pSP                  4       ^8X  dp   SP                  4       ^8X  d[   \         P                  ! SP                  \         P                  8H  ;'       d    SP                  \         P                  8H  R 4       EMV'       EdC   SP                  \         P                  8X  d   ^pS^,          oM*^ pS P                  \         P                  8X  d
   S^,          o^p\        SV4      p\        V^4      ^,          pV\        SV4      ,          V,          oV\        SV4      ,          V,          oSP                  4       S8X  dd   SP                  4       S8X  dO   \         P                  ! SP                  4       R 4       \         P                  ! SP                  4       R 4       EM\         P                  ! RVVVV3R l4       EM\         P                  ! SP                  \         P                  8H  ;'       d    SP                  \         P                  8H  R 4       \         P                  ! SP                  4       ^8H  ;'       d    SP                  4       ^8H  VV3R l4       SP                  ^ 4      S8X  d   SP                  ^4      ^8X  dm   SP                  ^ 4      ^8X  dW   SP                  ^4      S8X  dA   \         P                  ! SP                  4       ;'       d    SP                  4       R 4       EMhSP                  ^ 4      S8X  d\   SP                  ^4      SP                  ^ 4      u;8X  d   \        S^4      8X  d%   M M!SP                  ^4      \        S^4      8X  d   MSP                  ^ 4      S8X  dR   SP                  ^4      SP                  ^ 4      u;8X  d   \        S^4      8X  d   M MSP                  ^4      S8X  d   MSP                  ^ 4      \        S^4      8X  dR   SP                  ^4      SP                  ^ 4      u;8X  d   \        S^4      8X  d   M MSP                  ^4      S8X  d   M\         P                  ! RVVVVV3R l4       Ve   TMS P                  p\         P                  ! S P                  ^ 4      SP                  ^4      VS P                   R7      # )c                     V \         P                  \         P                  \         P                  \         P                  \         P
                  39   # rH   r]   r  float8_e5m2float8_e4m3fnuzfloat8_e5m2fnuzfloat4_e2m1fn_x2r   s   &r7   is_fp8_or_fp4_type2_check_scaled_mm_sizes.<locals>.is_fp8_or_fp4_type|  A    !!!!""
 
 	
r9   c                  L   < R SP                  4        RS P                  4        2# z%Inputs must be 2D but got self.dim()=z and mat2.dim()=r  r  r   s   r7   rl   (_check_scaled_mm_sizes.<locals>.<lambda>  %    7
|CSTXT\T\T^S_`r9   c                  <   < R SP                    RS P                    2# z?Expected both inputs to be fp8 or fp4 types but got self.dtype=z and mat2.dtype=r   r	  s   r7   rl   r	    $    QRVR\R\Q]]mnrnxnxmyzr9   r   r  r   c                 P    V ^ ,          V ^,          8  ;'       d    V ^,          ^8H  # r  r3   r  s   &r7   is_row_major,_check_scaled_mm_sizes.<locals>.is_row_major  %    !9vay(;;VAY!^;r9   c                 B    V ^ ,          ^8H  ;'       d    V ^,          ^8  # r  r3   r  s   &r7   is_col_major,_check_scaled_mm_sizes.<locals>.is_col_major       !9>33fQi!m3r9   c                 b    V P                  ^ 4      ^ 8H  ;'       g    V P                  ^4      ^ 8H  # r  r   	tensor_2ds   &r7   has_zero_dim,_check_scaled_mm_sizes.<locals>.has_zero_dim  ,    >>!$)CCY^^A->!-CCr9   c                  *   < R S P                  4        2# z#self must be row_major, got stride r  r   s   r7   rl   r	        =dkkm_Mr9   c                  *   < R S P                  4        2# z#mat2 must be col_major, got stride r  r  s   r7   rl   r	    r	  r9   c                  ,   < R S P                  ^4       2# zBExpected self.size(1) to be divisible by 16, but got self.size(1)=r   r   s   r7   rl   r	        \]a]f]fgh]i\jkr9   c                  "   < R S P                    2# z?Expected both dimensions of mat2 to be divisible by 16 but got r8  r 	  s   r7   rl   r	        YZ^ZdZdYefr9   c                      R # )zNFor tensorwise scaling, both scale_a and scale_b must be float (fp32) tensors.r3   r3   r9   r7   rl   r	    s    hr9   c                      R # )zscale_a must be contiguousr3   r3   r9   r7   rl   r	        8r9   c                      R # )zscale_b must be contiguousr3   r3   r9   r7   rl   r	    r)	  r9   Fc            	      Z   < R S  RSP                  4        RS RSP                  4        R2	# )zTInvalid blockwise scaling configuration. For blockwise scaling, scale_a should have  elements, got z, scale_b should have r  r  )expected_a_sizeexpected_b_sizer  r  s   r7   rl   r	    sC    FFUEVVefmfsfsfuev w//>.?w}}N__`br9   c                      R # )zKFor rowwise scaling, both scale_a and scale_b must be float (fp32) tensors.r3   r3   r9   r7   rl   r	    s    er9   c                  P   < R S P                  4       : RSP                  4       : 2# )zLFor non-tensorwise scaling, scale tensors must be 2D, but got scale_a.dim()=z and scale_b.dim()=r  r  r  s   r7   rl   r	    s*    gY`YdYdYfXhh|nunynyn{m}~r9   c                      R # )z@Both scale_a and scale_b must be contiguous for rowwise scaling.r3   r3   r9   r7   rl   r	    s    ^r9   c                    < R S RS RS R\        S ^4       R2	R\        S ^4       R\        S^4       RS R\        S ^4       R2	,           R\        S ^4       RS R\        S^4       R\        S ^4       R2	,           R\        S ^4       RS RSP                  ^ 4       RSP                  ^4       R	SP                  ^ 4       RSP                  ^4       R
2,           # )}Invalid scaling configuration. For tensorwise scaling, both scales should be scalar. For rowwise scaling, scale_a should be (, 1), scale_b should be (1, >). For (BlockWise1x128, BlockWise128x128), scale_a should be (r   ), scale_b should be (<). For (BlockWise1x128, BlockWise1x128), scale_a should be (z>). For (BlockWise128x128, BlockWise1x128), scale_a should be (). Got scale_a.size()=() and scale_b.size()=(r   r8   r   )_kr  r  r  r  s   r7   rl   r	  #  sG   CCD#Eabcad eVVWUXXZ[cdfhk[lZmmpr 0S0A/B"XaQTEUDV WTTUSVVXYabdfiYjXkknp	p 0S0A/B"QC HVV^_`beVfUggijrsuwzj{i||AA 0S0A/B"QC H//6||A.?r',,q/AR S//6||A.?r',,q/ARRSU
Ur9   r  )r]   ro   r   re   r   r   r   r   float8_e8m0fnur  r   r  r	  r8   r  r   r   )r   r  r  r  r  r  r  r  r	  r	  r	  r	  is_blockwise_scalingblock_size_kblock_size_mnnum_k_blockspadded_num_k_blocks
_out_dtyper=	  r-	  r.	  r  r  s   ffff&&&&          @@@@@r7   _check_scaled_mm_sizesrE	  r  st   
 
LL
a++DHHJ!O` 
LL4::&II+=djj+Iz 	DV#t%t%	<	4	D t%LLT[[]+AA|D/AM LLT[[]+AA|D/AM LL		!r!Q&k LL		!r!Q&AA499Q<"+<+Af 

2IIaL !5!55 : :MMU%9%99 
 !4!44 9 9MMU%8%88 	 ==?aGMMOq$8LL.QQ7==EMM3Qh "! }} 3 33  "!V!::!7!77aBM#B5L"*<";a"? M ::=PP  M ::=PP 
 ?2MMO6))+8 ))+8
  LL.QQ7==EMM3Qe
 LL"99w{{}'9~ Q1$LLOq(LLOq(LLOq( ))+GG0E0E0G^
 Q1$LLOw||AK(2s:KKLLOx3'77 Q1$LLOw||AK(2s:KKLLOq( Q8As#33LLOw||AK(2s:KKLLOq(   " (3J;;tyy|TYYq\DKKXXr9   c                .   V ^8  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R	\        /# r  r  )rC   s   "r7   rD   rD   7  s      
,,
,, \\ \\	
 ,,
 ,,% {{T! r9   c           
          \        WW#WEWg4      # rH   )rE	  )r   r  r  r  r  r  r  r  s   &&&&&&&&r7   meta_scaled_mmrH	  6  s     "Gd) r9   c                   V ^8  d   QhR\         P                  R\         P                  R\        \         P                  ,          R\        \        ,          R\        \         P                  ,          R\        \        ,          R\         P                  R,          R	\         P                  R,          R
\        \
        ,          R,          R\        \
        ,          R,          R\        /# )rA   r   r  r  scale_recipe_ar  scale_recipe_br  Nr  	swizzle_a	swizzle_br  )r]   r   r   r(   re   r)   r   )rC   s   "r7   rD   rD   F  s     iC iC
,,iC
,,iC %,,iC %	iC
 %,,iC %iC ,,
iC {{T!iC K 4'iC K 4'iC iCr9   c           
      !  a aaaaa	aaa a!a"a#a$a%a& R  pR p\         P                  ! S P                  4       ^8H  ;'       d    SP                  4       ^8H  VV 3R l4       \         P                  ! V! S P                  4      ;'       d    V! SP                  4      VV 3R l4       S P                  ^ ,          o S P                  ^,          oSP                  ^,          o!V! S P                  4      '       d$   V! SP                  4      '       d   ^pSV,          oV Uu. uF  p\        V4      NK  	  ppV Uu. uF  p\        V4      NK  	  ppS'       d   S Uu. uF  p\        V4      NK  	  upoM\        P                  .oS	'       d   S	 Uu. uF  p\        V4      NK  	  upo	M\        P                  .o	\        S 4      R8X  g"   \        S 4      R8X  g   \        S 4      R8X  Ed   R pR pR	 p\        S 4      R8w  Ed
   \         P                  ! V! S P                  4       4      ;'       g	    V! S 4      V 3R
 l4       \         P                  ! V! SP                  4       4      ;'       g	    V! S4      V3R l4       \         P                  ! S P                  ^4      ^,          ^ 8H  V 3R l4       \         P                  ! SP                  ^ 4      ^,          ^ 8H  ;'       d    SP                  ^4      ^,          ^ 8H  V3R l4       R R lpR R lpR R lpR R lpR R lpR R lpR R lpR R lpV! W54      '       d   \         P                  ! S^ ,          P                  4       ^8H  ;'       ds    S^ ,          P                  4       ^8H  ;'       dQ    S^ ,          P                  \         P                  8H  ;'       d%    S^ ,          P                  \         P                  8H  R 4       EMV! W54      '       d   \         P                  ! S^ ,          P                  ^ ,          S 8H  ;'       d    S^ ,          P                  4       S 8H  ;'       ds    S^ ,          P                  \         P                  8H  ;'       dG    S^ ,          P                  4       S!8H  ;'       d%    S^ ,          P                  \         P                  8H  VVVV 3R l4       EMV! W54      '       Ed   S^ ,          P                  \         P                  8H  ;'       d%    S^ ,          P                  \         P                  8H  pS^ ,          o%S%P                  ^ ,          S 8H  ;'       d    S%P                  ^,          S^,          8H  ;'       dk    S%P                  ^ 4      ^8H  ;'       dO    S%P                  ^4      S 8H  ;'       g3    S%P                  ^,          ^8H  ;'       d    S%P                  ^4      ^8H  pS^ ,          o&S&P                  ^ ,          S!8H  ;'       d    S&P                  ^,          S^,          8H  ;'       dk    S&P                  ^ 4      ^8H  ;'       dO    S&P                  ^4      S!8H  ;'       g3    S&P                  ^,          ^8H  ;'       d    S&P                  ^4      ^8H  p\         P                  ! T;'       d    T;'       d    TVV V!V%V&3R  l4       E	MV! W54      '       Ed   S^ ,          P                  \         P                  8H  ;'       d%    S^ ,          P                  \         P                  8H  p\        S^,          ^4      oS^ ,          o%S%P                  ^ ,          S8H  ;'       d    S%P                  ^,          S ^,          8H  ;'       dk    S%P                  ^ 4      ^8H  ;'       dO    S%P                  ^4      S8H  ;'       g3    S%P                  ^,          ^8H  ;'       d    S%P                  ^4      ^8H  pS^ ,          o&S&P                  ^ ,          S!8H  ;'       d    S&P                  ^,          S^,          8H  ;'       dk    S&P                  ^ 4      ^8H  ;'       dO    S&P                  ^4      S!8H  ;'       g3    S&P                  ^,          ^8H  ;'       d    S&P                  ^4      ^8H  p\         P                  ! T;'       d    T;'       d    TVVV V!V%V&3R! l4       EMV! W54      '       Ed   S^ ,          P                  \         P                  8H  ;'       d%    S^ ,          P                  \         P                  8H  p\        S^,          ^4      oS^ ,          o%S%P                  ^ ,          S 8H  ;'       d    S%P                  ^,          S^,          8H  ;'       dk    S%P                  ^ 4      ^8H  ;'       dO    S%P                  ^4      S 8H  ;'       g3    S%P                  ^,          ^8H  ;'       d    S%P                  ^4      ^8H  pS^ ,          o&S&P                  ^ ,          S8H  ;'       d    S&P                  ^,          S!^,          8H  ;'       dk    S&P                  ^ 4      ^8H  ;'       dO    S&P                  ^4      S8H  ;'       g3    S&P                  ^,          ^8H  ;'       d    S&P                  ^4      ^8H  p\         P                  ! T;'       d    T;'       d    TVVV V!V%V&3R" l4       EMyV! W54      '       Ed   \         P                  P                  '       d|   \!        S P                  ^ ,          ^ 4      S P                  ^,          ,          o"\!        S P                  ^,          ^ 4      S P                  ^ ,          ,          o#\        P                  o$M\        S P                  ^ ,          ^4      \        \!        S P                  ^,          ^ 4      ^4      ,          o"\        SP                  ^,          ^4      \        \!        S P                  ^,          ^ 4      ^4      ,          o#\        P"                  o$\         P                  ! S^ ,          P                  4       S"8H  ;'       d    S^ ,          P                  \         P$                  8H  ;'       do    S^ ,          P                  4       S#8H  ;'       dM    S^ ,          P                  \         P$                  8H  ;'       d!    S^ ,          S$8H  ;'       d    S	^ ,          S$8H  V"V#V$VVVV	3R# l4       EMPV! W54      '       Ed<   \        S ^4      \        \!        S^4      ^4      ,          o"\        S!^4      \        \!        S^4      ^4      ,          o#\        P"                  o$\         P                  ! S^ ,          P                  4       S"8H  ;'       d    S^ ,          P                  \         P&                  8H  ;'       do    S^ ,          P                  4       S#8H  ;'       dM    S^ ,          P                  \         P&                  8H  ;'       d!    S^ ,          S$8H  ;'       d    S	^ ,          S$8H  V"V#V$VVVV	3R$ l4       EMV! W54      '       Ed   \        S ^4      \        \!        S^4      ^4      ,          o"\        S!^4      \        \!        S^4      ^4      ,          o#\        P"                  o$\         P                  ! S^ ,          P                  4       S"8H  ;'       Ed8    S^ ,          P                  \         P&                  8H  ;'       Ed    S^,          P                  4       ^8H  ;'       d    S^,          P                  \         P                  8H  ;'       d    S^ ,          P                  4       S#8H  ;'       d    S^ ,          P                  \         P&                  8H  ;'       do    S^,          P                  4       ^8H  ;'       dM    S^,          P                  \         P                  8H  ;'       d!    S^ ,          S$8H  ;'       d    S	^ ,          S$8H  V"V#V$VVVV	3R% l4       M\         P                  ! R&VV V!VV3R' l4       Ve   TMS P                  p\         P(                  ! S S!VS P*                  R(7      # u upi u upi u upi u upi ))c                     V \         P                  \         P                  \         P                  \         P                  \         P
                  39   # rH   r  r   s   &r7   r	  5_check_scaled_mm_sizes_v2.<locals>.is_fp8_or_fp4_typeS  r	  r9   c                 (    V \         P                  8H  # rH   )r]   r	  r   s   &r7   is_fp4_type._check_scaled_mm_sizes_v2.<locals>.is_fp4_type\  s    ....r9   c                  L   < R SP                  4        RS P                  4        2# r	  r  r	  s   r7   rl   +_check_scaled_mm_sizes_v2.<locals>.<lambda>a  r		  r9   c                  <   < R SP                    RS P                    2# r	  r   r	  s   r7   rl   rU	  e  r	  r9   r   r  r   c                 P    V ^ ,          V ^,          8  ;'       d    V ^,          ^8H  # r  r3   r  s   &r7   r	  /_check_scaled_mm_sizes_v2.<locals>.is_row_major  r	  r9   c                 B    V ^ ,          ^8H  ;'       d    V ^,          ^8  # r  r3   r  s   &r7   r	  /_check_scaled_mm_sizes_v2.<locals>.is_col_major  r	  r9   c                 b    V P                  ^ 4      ^ 8H  ;'       g    V P                  ^4      ^ 8H  # r  r   r	  s   &r7   r	  /_check_scaled_mm_sizes_v2.<locals>.has_zero_dim  r	  r9   c                  *   < R S P                  4        2# r	  r  r   s   r7   rl   rU	    r	  r9   c                  *   < R S P                  4        2# r	  r  r 	  s   r7   rl   rU	    r	  r9   c                  ,   < R S P                  ^4       2# r"	  r   r   s   r7   rl   rU	    r#	  r9   c                  "   < R S P                    2# r%	  r8  r 	  s   r7   rl   rU	    r&	  r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rA   recipe_arecipe_br   r(   )rC   s   "r7   rD   /_check_scaled_mm_sizes_v2.<locals>.__annotate__  s$     	 	D$5 	kAR 	r9   c                     \        V 4      ^8H  ;'       dS    \        V4      ^8H  ;'       d=    V ^ ,          \        P                  8H  ;'       d    V^ ,          \        P                  8H  # r1   )r   r(   
TensorWiserc	  rd	  s   &&r7   is_tensorwise0_check_scaled_mm_sizes_v2.<locals>.is_tensorwise  sc    H" : :MQ&: :QK;#9#99: : QK;#9#99	r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rb	  re	  )rC   s   "r7   rD   rf	    s$     	 	k!2 	d;>O 	r9   c                     \        V 4      ^8H  ;'       dS    \        V4      ^8H  ;'       d=    V ^ ,          \        P                  8H  ;'       d    V^ ,          \        P                  8H  # r1   )r   r(   RowWiseri	  s   &&r7   
is_rowwise-_check_scaled_mm_sizes_v2.<locals>.is_rowwise  sc    H" 7 7MQ&7 7QK;#6#667 7 QK;#6#66	r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rb	  re	  )rC   s   "r7   rD   rf	    s$     	 	D- 	k9J 	r9   c                     \        V 4      ^8H  ;'       dS    \        V4      ^8H  ;'       d=    V ^ ,          \        P                  8H  ;'       d    V^ ,          \        P                  8H  # r1   )r   r(   BlockWise1x32ri	  s   &&r7   is_mx(_check_scaled_mm_sizes_v2.<locals>.is_mx  sc    H" = =MQ&= =QK;#<#<<= = QK;#<#<<	r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rb	  re	  )rC   s   "r7   rD   rf	    s&     	 	;'	373D	r9   c                     \        V 4      ^8H  ;'       dS    \        V4      ^8H  ;'       d=    V ^ ,          \        P                  8H  ;'       d    V^ ,          \        P                  8H  # r1   )r   r(   BlockWise1x16ri	  s   &&r7   is_nv_single_level5_check_scaled_mm_sizes_v2.<locals>.is_nv_single_level  se     H" = =MQ&= =QK;#<#<<= = QK;#<#<<	r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rb	  re	  )rC   s   "r7   rD   rf	    s$     	 	D- 	k9J 	r9   c                 Z   \        V 4      ^8H  ;'       d    \        V4      ^8H  ;'       d    V ^ ,          \        P                  8H  ;'       d_    V ^,          \        P                  8H  ;'       d=    V^ ,          \        P                  8H  ;'       d    V^,          \        P                  8H  # rO  )r   r(   rx	  rh	  ri	  s   &&r7   is_nv(_check_scaled_mm_sizes_v2.<locals>.is_nv  s    H" : :MQ&: :QK;#<#<<: : QK;#9#99: : QK;#<#<<	: :
 QK;#9#99r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rb	  re	  )rC   s   "r7   rD   rf	    s$     	 	T+%6 	${BS 	r9   c                     \        V 4      ^8H  ;'       dS    \        V4      ^8H  ;'       d=    V ^ ,          \        P                  8H  ;'       d    V^ ,          \        P                  8H  # r1   )r   r(   BlockWise1x128ri	  s   &&r7   is_1x128_1x1281_check_scaled_mm_sizes_v2.<locals>.is_1x128_1x128  sc    H" > >MQ&> >QK;#=#==> > QK;#=#==	r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rb	  re	  )rC   s   "r7   rD   rf	    $     	 	tK'8 	DDU 	r9   c                     \        V 4      ^8H  ;'       dS    \        V4      ^8H  ;'       d=    V ^ ,          \        P                  8H  ;'       d    V^ ,          \        P                  8H  # r1   )r   r(   r	  BlockWise128x128ri	  s   &&r7   is_1x128_128x1283_check_scaled_mm_sizes_v2.<locals>.is_1x128_128x128  si    H" @ @MQ&@ @QK;#=#==@ @ QK;#?#??	r9   c                \    V ^8  d   QhR\         \        ,          R\         \        ,          /# rb	  re	  )rC   s   "r7   rD   rf	    r	  r9   c                     \        V 4      ^8H  ;'       dS    \        V4      ^8H  ;'       d=    V ^ ,          \        P                  8H  ;'       d    V^ ,          \        P                  8H  # r1   )r   r(   r	  r	  ri	  s   &&r7   is_128x128_1x1283_check_scaled_mm_sizes_v2.<locals>.is_128x128_1x128  sc    H" > >MQ&> >QK;#?#??> > QK;#=#==	r9   c                      R # )z\For Tensorwise scaling, both scale_a and scale_b must be single element float (fp32) tensorsr3   r3   r9   r7   rl   rU	    s    vr9   c            	         < R SP                   ^ ,           RS^ ,          P                  4        RS P                   ^,           RS^ ,          P                  4        R2	# )z'For Rowwise scaling, scale_a must have z elements (got: z), and scale_b must have r   )r   r   )r  r  r  r   s   r7   rl   rU	    s\    =djjm_L\]def]g]m]m]o\p q//3zz!}o=MgVWjN^N^N`Maabdr9   c                     < R S RS ^,           RSP                    RS RSP                   RS RS ^,           RSP                    RS RSP                   R2# )z>For 1x128 x 1x128 blockwise scaling, scale a must have shape [r   ] (got: ) and stride [1, )scale b must have shape [r   r   r   )rW  r4  r4  sasbs   r7   rl   rU	    sy    001s"Q#XJhrxxjPabcaddlmomvmvlw x001s"Q#XJhrxxjPabcaddlmomvmvlwwxzr9   c                     < R S RS^,           RSP                    RS RSP                   RS RS ^,           RSP                    RS RSP                   R2# )z]For 128x128 x 1x128 blockwise scaling, L4 = {round_up(K / 128, 4)}, scale a must have shape [r   r	  r	  r	  r   r	  rW  L4r4  r4  r	  r	  s   r7   rl   rU	  7  sy    002t2a3hZxzQbcebffnoqoxoxny z001s"Q#XJhrxxjPabcaddlmomvmvlwwxzr9   c                     < R S RS ^,           RSP                    RS RSP                   RS RS^,           RSP                    RS RSP                   R2# )z]For 1x128 x 128x128 blockwise scaling, L4 = {round_up(K / 128, 4)}, scale a must have shape [r   r	  r	  r	  r   r	  r	  s   r7   rl   rU	  V  sy    001s"Q#XJhrxxjPabcaddlmomvmvlw x002t2a3hZxzQbcebffnoqoxoxnyyz|r9   c                    < R S  RS^ ,          P                  4        RS RS^ ,          P                  4        R\        P                   RS^ ,          P                   RS^ ,          P                   RS RS^ ,           RS^ ,           R2# )	z!for MX scaling scale_a must have  (got: ) and scale_b must have z). Scales must have types z (for self: 	, mat_b: z) Must have swizzle type  (got self: r   )r   r]   r>	  re   expected_scale_a_elemsexpected_scale_b_elemsexpected_swizzler  r  rL	  rM	  s   r7   rl   rU	  q  s    78N7OwW^_`WaWgWgWiVj k--C,DGGTUJL\L\L^K_ `""'"6"6!7|GAJDTDTCUU^_fgh_i_o_o^p q..>-?|IVWL>Ybclmncobppqsr9   c                     < R S  RS^ ,          P                  4        RS RS^ ,          P                  4        RS RS^ ,           RS^ ,           R2# )z.for single-level NV scaling scale_a must have r	  r	  ). Must have swizzle type r	  r	  r   r  r	  s   r7   rl   rU	    sr    DE[D\\cdklmdndtdtdvcw x--C,DGGTUJL\L\L^K_ `$$4#5\)A,yYbcdYeXffgir9   c                     < R S  RS^ ,          P                  4        RS RS^ ,          P                  4        RS RS^ ,           RS^ ,           R2# )z!for NV scaling scale_a must have r	  r	  r	  r	  r	  r   r  r	  s   r7   rl   rU	    sq    78N7OwW^_`WaWgWgWiVj k--C,DGGTUJL\L\L^K_ `$$4#5\)A,yYbcdYeXffgir9   Fc                    < R S RS RS R\        S ^4       R2	R\        S ^4       R\        S^4       RS R\        S ^4       R2	,           R\        S ^4       RS RS^ ,          P                  ^ 4       RS^ ,          P                  ^4       RS^ ,          P                  ^ 4       RS^ ,          P                  ^4       R	2,           # )
r4	  r5	  r6	  r   r7	  r8	  r9	  r:	  r;	  r   r<	  )rW  r4  r4  r  r  s   r7   rl   rU	    s   ??@cA]^_]` aRRSQTTVW_`acfWgVhhkm ,HQ,<+=RC@P?Q RPPQsRTU]^_adUeTffik	k ,HQ,<+=Rs C++21:??1+=*>bQRAS@T U++21:??1+=*>bQRAS@TTUWWr9   r  )r]   ro   r   re   r   r(   r)   
NO_SWIZZLEr   r   r   r   r  r>   r]  r^  r8   SWIZZLE_32_4_4r>	  r  r   r   )'r   r  r  rJ	  r  rK	  r  r  rL	  rM	  r  r	  rR	  K_packed_multipliersir	  r	  r	  rj	  ro	  rt	  ry	  r}	  r	  r	  r	  types_ok
scale_a_ok
scale_b_okrD	  rW  r	  r4  r4  r	  r	  r	  r	  r	  s'   fff&f&&&ff&                   @@@@@@@@@r7   _check_scaled_mm_sizes_v2r	  F  s   
/ 
LL
a++DHHJ!O` 
LL4::&II+=djj+Iz 	

1A

1A

1A 4::;tzz#:#:	  0>?"k"oN?0>?"k"oN?/89y[_y9	 ""
	 /89y[_y9	 ""
	
 	DV#t%t%	<	4	D t%LLT[[]+AA|D/AM LLT[[]+AA|D/AM LL		!r!Q&k LL		!r!Q&AA499Q<"+<+Af
								 88LL
  "a' 6 6AJ$$&!+6 6AJ$$56 6 AJ$$5v 77LL
  #q( 6 6AJ$$&!+6 6AJ$$56 6 AJ$$&!+6 6 AJ$$5
 N;;
 
  EMM1WWgaj6F6F%--6W  Bq  T THHQK18+T TIIaLA%T T YYq\Q&RR288A;!+;+Q+Q		!PQ@Q	  Bq  T THHQK18+T TIIaLA%T T YYq\Q&RR288A;!+;+Q+Q		!PQ@Q	  LL66Z66J  n== 
  EMM1WWgaj6F6F%--6W  !c'1%BBr! U UHHQK18+U UIIaLA%U U YYq\R'SSBHHQK1,<,R,R1QRAR	  Bq  T THHQK18+T TIIaLA%T T YYq\Q&RR288A;!+;+Q+Q		!PQ@Q	  LL66Z66J  n== 
  EMM1WWgaj6F6F%--6W  !c'1%BBq  T THHQK18+T TIIaLA%T T YYq\Q&RR288A;!+;+Q+Q		!PQ@Q	  Br! U UHHQK18+U UIIaLA%U U YYq\R'SSBHHQK1,<,R,R1QRAR	  LL66Z66J  >22}}   )1$**Q-)DtzzRS})T&)1$**Q-)DtzzRS})T&#.#9#9 )1$**Q-)ETZZ]B/I *& *2$**Q-)ETZZ]B/I *& $/#=#= LL
  "&<< 5 5AJ$$(<(<<5 5AJ$$&*@@5 5 AJ$$(<(<<5 5 aL$44	5 5
 aL$44   ??%-a%5!RRS8T%T"%-a%5!RRS8T%T"*99LL
  "&<< 5 5AJ$$(;(;;5 5AJ$$&*@@5 5 AJ$$(;(;;5 5 aL$44	5 5
 aL$44  >22%-a%5!RRS8T%T"%-a%5!RRS8T%T"*99LL
  "&<< 	5 	5AJ$$(;(;;	5 	5AJ$$&!+	5 	5 AJ$$5	5 	5 AJ$$&*@@		5 	5
 AJ$$(;(;;	5 	5 AJ$$&!+	5 	5 AJ$$5	5 	5 aL$44	5 	5 aL$44 $ LL
 
 (3J;;q!:dkkBBw	 @? : :s   AB06AB5AB:AB?c                   V ^8  d   QhR\         P                  R\         P                  R\        \         P                  ,          R\        \        ,          R\        \        ,          R\        \         P                  ,          R\        \        ,          R\        \        ,          R	\         P                  R
,          R\         P
                  R
,          R\        \        ,          R
,          R\        /# )rA   r   r  r  rJ	  rL	  r  rK	  rM	  r  Nr  contraction_dimsr  )r]   r   r   r(   r)   re   r   r   )rC   s   "r7   rD   rD     s      
,,
,, %,, %	
 K  %,, % K  ,,
 ++$ 3i$& r9   c                 .    \        V VVVVVVV	VVVR 7      # ))r  r  rL	  rM	  r  )r	  )r   r  r  rJ	  rL	  r  rK	  rM	  r  r  r	  r  s   &&&&&&&&&&&&r7   meta_scaled_mm_v2r	    s2     %% r9   c           	      V    \        WW#VR R7       V P                  V P                  4      # T)r*  rF  r   r   r   ru  r  r  s   &&&&&&r7   meta_scatter_reduce_twor	    s%     dVTJ>>$**%%r9   c           	      $    \        WW#VR R7       V # r	  rJ  r	  s   &&&&&&r7   meta_scatter_reduce__twor	    s    dVTJKr9   c                  a  \         P                  ! ^ S P                  4       u;8  ;'       d    ^8*  Mu V 3R l4       S P                  4       ^8X  d2   \         P                  ! V\         P                  S P
                  R7      # \         P                  ! S P                  ^ 4      V\         P                  S P
                  R7      # )r   c                  *   < R S P                  4        2# )zAThe probability distributions dimensions must be 1 or 2, but got r  r  s   r7   rl   "meta_multinomial.<locals>.<lambda>  s    STYT]T]T_S`ar9   r  )r]   ro   r   r   r   r   r   )r   num_samplesreplacementr  s   f&&$r7   meta_multinomialr	    s     
LL	EIIK1a yy{a{{;ejjNN;;

1{%**U\\ r9   c                 ,    ^pV  F  pW,          pK  	  V# r1   r3   )vsr  vs   &  r7   multiply_integersr	    s    	A	 Hr9   c                   a aaa \         P                  ! \        S4      S8H  VV3R  l4       S^,           o\         P                  ! \        S 4      S8H  VV 3R l4       \         P                  ! \        ;QJ d#    R S R,           4       F  '       d   K   RM	  RM! R S R,           4       4      ;'       d5    \        ;QJ d    R S 4       F  '       d   K   RM	  RM! R S 4       4      V V3R l4       S R,          w  r4W4.SO5# )	c                  &   < R S  R\        S4       2# )z%It is expected output_size equals to , but got size r_  )num_spatial_dimsrB  s   r7   rl   'upsample_common_check.<locals>.<lambda>  s    78H7IY\]hYiXjkr9   c                  &   < R S  R\        S4       2# )z$It is expected input_size equals to r	  r_  )expected_input_dimsr  s   r7   rl   r	    s    67J6K?[^_i[jZklr9   c              3   *   "   T F	  q^ 8  x  K  	  R# 5ir   Nr3   r{   r  s   & r7   r}   (upsample_common_check.<locals>.<genexpr>  s     *>aE>   rU  FTc              3   *   "   T F	  q^ 8  x  K  	  R# 5ir	  r3   r	  s   & r7   r}   r	    s     2N+Qq5+r	  c                     < R S  RS 2# )zDInput and output sizes should be greater than 0, but got input size z and output size r3   )r  rB  s   r7   rl   r	    s      \!2;-Ar9   :NrA   N)r]   ro   r   rZ  )r  rB  r	  ry  channelsr	  s   fff  @r7   upsample_common_checkr	    s    	LLK,,k +Q.	LLJ..l
 
LL*:b>**:b>**NNss2N+2Nsss2N+2N/N	A ""~F+{++r9   c                 N  a  \         P                  ! S P                  4       ^ 8g  ;'       g!    \        S P	                  4       R,          4      V 3R l4       \        S P	                  4       V^R7      pS P                  V4      P                  \        P                  ! S 4      R7      # )r   r2   NNc                  *   < R S P                  4        2# )z>Non-empty 3D data tensor expected but got a tensor with sizes r   r  s   r7   rl   $upsample_nearest1d.<locals>.<lambda>      PQVQ[Q[Q]P^_r9   r	  r   
r]   ro   r   r	  r   r	  r   rr  rV   r   )r   rB  scalesfull_output_sizes   f&& r7   upsample_nearest1dr	         
LLAA/

R0@A_ -

kA ??+,//11%8 0  r9   c                   a  \         P                  ! S P                  4       ^ 8g  ;'       g!    \        S P	                  4       R,          4      V 3R l4       \        S P	                  4       V^R7      pS P                  V4      p\        P                  ! S 4      pS P                  w  rx pS P                  P                  R8X  d   V^8  d   \         P                  pVP                  VR7      pV# )r   r	  c                  *   < R S P                  4        2# z>Non-empty 4D data tensor expected but got a tensor with sizes r   r  s   r7   rl   $upsample_nearest2d.<locals>.<lambda>  r	  r9   r	  r   r   )r]   ro   r   r	  r   r	  r   rV   r   r   r   r   r   
contiguous)	r   rB  scales_hscales_wr	  r  r   rY   
n_channelss	   f&&&     r7   upsample_nearest2dr	    s     
LLAA/

R0@A_ -

kA __-.F //6M  ++A1a||F"zA~//];FMr9   c                    V ^8  d   QhR\         R\        \        \        P                  ,          ,          R\        \        \        P                  ,          ,          R\
        R,          R\
        R,          /# )rA   r  rB  r  r	  Nr	  )r   r   r   r]   r  ra   )rC   s   "r7   rD   rD   7  s^      #,- u||+, dl	
 dlr9   c                 l  a aa \        W!^R7      o\        P                  ! S P                  ^8H  V 3R l4       \	        ^4       F9  o\        P                  ! S P                  S4      SS,          8H  VV V3R l4       K;  	  S P                  V4      P                  \        P                  ! S 4      R7      # )rA   r	  c                  "   < R S P                    2# zFExpected grad_output to be a tensor of dimension 4 but got: dimension rv  rn  s   r7   rl   -upsample_nearest2d_backward.<locals>.<lambda>C      XYdYiYiXjkr9   c            
      L   < R S RS S,           RS RSP                  S4       2# )zCExpected grad_output to have the same shape as output; output.size() = z but got grad_output.size(r   r	  r  r   s   r7   rl   r	  H  s9      !s$'7':&;,QCtK4D4DQ4G3HJr9   r   )
r	  r]   ro   r   r   r   r   rr  rV   r   )r  rB  r  r	  r	  r	  r   s   f&&&&@@r7   upsample_nearest2d_backwardr	  1  s     -! 
LLAk 1XQ#3A#66	
    ,//11+> 0  r9   c                 N  a  \         P                  ! S P                  4       ^ 8g  ;'       g!    \        S P	                  4       R,          4      V 3R l4       \        S P	                  4       V^R7      pS P                  V4      P                  \        P                  ! S 4      R7      # )r   r	  c                  *   < R S P                  4        2# )z>Non-empty 5D data tensor expected but got a tensor with sizes r   r  s   r7   rl   $upsample_nearest3d.<locals>.<lambda>Z  r	  r9   r	  r   r	  )r   rB  scales_dr	  r	  r	  s   f&&&& r7   upsample_nearest3dr	  T  r	  r9   c                     \         P                  ! V 4      \         P                  ! V \         P                  R 7      rvVe   Ve   \        V\        4      '       g   \        R\        V4       24      h\        V\        4      '       g   \        R\        V4       24      hVP                  pVP                  4       p	\        WH4      p\        WX4      pVP                  W4       VP                  W4       \        WdR7       \        WuR7       WE3# Wg3# )r   zvalues must be TensorLike, got z indices must be TensorLike, got )r  r  )r]   r   r   ry   r   r   r   r   r   r!   r   r#   )
r   stabler   
descendingr   r   r	  r   r   
out_strides
   &&&&&&    r7   	meta_sortr	  d  s     D!5#3#3D#Lqg1&*-- #B4<.!QRR':.. #CDM?!STT GG	XXZ
"65#G791I2344Kr9   c           	        a aaaaaaa \         P                  ! S P                  ^8H  V 3R l4       \         P                  ! S P                  SP                  8H  VV 3R l4       S P	                  ^4      oSe   \         P                  ! SP                  ^8H  V3R l4       \         P                  ! SP                  4       S8H  VV3R l4       \         P                  ! SP                  SP                  8H  VV3R l4       \         P                  ! SP                  ^8H  V3R l4       S P	                  ^ 4      S,          S,          o\         P                  ! SP                  4       S8H  VVVV V3R l4       \         P                  ! \        ;QJ d#    V 3R	 lSSSS3 4       F  '       d   K   R
M	  RM! V 3R	 lSSSS3 4       4      R 4       R# )rA   c                  "   < S P                    R 2# z != 2rv  )input_gatess   r7   rl   %rnn_cell_checkSizes.<locals>.<lambda>      ;3C3C2DE0Jr9   c                  :   < SP                    R S P                    2# r  r8  )hidden_gatesr	  s   r7   rl   r 
    s    ;$$%T,*<*<)=>r9   Nc                  "   < S P                    R 2# )z != 1rv  )
input_biass   r7   rl   r 
    s    joo5Fe3Lr9   c                  .   < SP                  4        R S  2# r
  r  )
gates_sizer
  s   r7   rl   r 
    s    z'')*$zl;r9   c                  :   < SP                    R S P                    2# r
  r8  )hidden_biasr
  s   r7   rl   r 
    s    z''([->->,?@r9   c                  "   < S P                    R 2# r	  rv  )prev_hiddens   r7   rl   r 
    r
  r9   c            
      `   < SP                  4        R SP                  ^ 4       RS RS RS  R2
# )r  z * z // z (aka r   )r   r   )expected_prev_hidden_numelfactorr
  r	  r
  s   r7   rl   r 
    s@    ;$$&'tK,<,<Q,?+@J<tTZS[[ab|a}}~r9   c              3   V   <"   T F  pVP                   SP                   8H  x  K   	  R # 5irH   rN  )r{   r<   r	  s   & r7   r}   &rnn_cell_checkSizes.<locals>.<genexpr>  s'      
I HH***Is   &)FTc                      R # )z%expected all inputs to be same devicer3   r3   r9   r7   rl   r 
    s    7r9   )r]   ro   r   r   r   r   rZ  )r	  r
  r
  r

  r
  r
  r
  r
  s   ffffff@@r7   rnn_cell_checkSizesr
    sa    
LL!!Q&(JK	LL\///> !!!$JZ__)+LM*,;	
 	 1 11@	
 
LL!!Q&(JK!,!1!1!!4z!AV!K	LL99 
LL 
"J[I
 
"J[I
 	
 	8r9   c                 
   \        WW4^V4       \        P                  ! V \        P                  R7      p\        P                  ! V\        P                  R7      p\        P                  ! V\        P                  R7      pWgV3# )r  r   )r
  r]   r   r   )r	  r
  cxr
  r

  	workspacehycys   &&&&&   r7   _thnn_fused_lstm_cell_metar
    se     :ArR  E<S<STI			"E,C,C	DB			"E,C,C	DBIr9   c                    \        V4      ^ 8g  pV'       d)   \        V4      pV^ ,          pV P                  ^ ,          pM^V
'       d   V P                  ^,          MV P                  ^ ,          pV
'       d   V P                  ^ ,          MV P                  ^,          pRpV'       d   ^M^pV^ 8w  d   TMTpV'       d   VVV,          .pM V
'       d   VVVV,          .MVVVV,          .pV P                  V4      pV	V,          VV.pVf$   \        P                  ! ^ V P
                  R7      pMVP                  V4      pVP                  V	V,          VV.4      pV'       d   ^ M^ pV P                  V\        P                  R7      pVVVVV3# )r   rN  r   r   )r   r   r   r]   r   r   r  )r   r  weight_stride0
weight_bufhxr
  r  hidden_size	proj_size
num_layersbatch_firstdropouttrainbidirectionalr?  dropout_stateis_input_packed
seq_length
mini_batchbatch_sizes_sumnum_directionsout_sizer   r  
cell_shaper
  r
  reserve_shapereserves   &&&&&&&&&&&&&&&&             r7   
_cudnn_rnnr/
    sS   & +&!+O%
 ^
++a.'2U[[^A
'2U[[^A
'QQN%NyH$h&?@	  X%>?j(^*CD 	
 __Y'F~-z;GJ	z[[5<<0\\*%	zN2JI	JB AAMoom5;;o?G2r7J..r9   c                     ^ pV F0  pVP                  4       ^ 8  g   K  WP                  4       ,          pK2  	  V P                  V34      p\        V VVVVVVV^ VVV	V
VVV4      # r  )r   r   r/
  )r   r  r
  r
  r
  r  r
  r 
  r!
  r"
  r#
  r$
  r?  r%
  total_weight_elemsr  r
  s   &&&&&&&&&&&&&&   r7   
miopen_rnnr2
    s    & 779q='')+  "4!67J

	! r9   c                 z   V'       d   V P                   ^,          MV P                   ^ ,          pV'       d   V P                   ^ ,          MV P                   ^,          pT
pV'       d   VVV.MVVV.pV P                  V4      pVf$   \        P                  ! ^ V P                  R7      pMVP                  VP                   4      pVf$   \        P                  ! ^ V P                  R7      pMVP                  VP                   4      p\        P                  ! ^ V P                  \        P
                  R7      pVVVV3# )r2   rN  r   )r   r   r]   r   r   r  )r   w0w1w2w3hx_cx_r   r?  r  r
  r 
  
has_biasesr$
  r!
  r#
  r'
  r(
  output_chanelsr   r  r
  r
  r
  s   &&&&&&&&&&&&&&&&        r7   mkldnn_rnn_layerr<
    s    & $/QEKKNJ#.QEKKNJ N  
Z0*n5 
 __Y'F
{[[5<<0]]399%
{[[5<<0]]399%Aell%++FI2r9$$r9   c                    aa V P                   ^ 8X  d/   \        P                  ! S^ 8H  ;'       g    SR8H  VV3R l4       R# \        P                  ! V P                  S4      ^ 8g  VV3R l4       R# )r   c                     < S R S  2# )z4: Expected reduction dim -1 or 0 for scalar but got r3   r   rC  s   r7   rl   'zero_numel_check_dims.<locals>.<lambda>H  s    wiSTWSXYr9   c                     < S R S  R2# )z: Expected reduction dim z to have non-zero size.r3   r?
  s   r7   rl   r@
  M  s    wi8=TUr9   Nr   )r   r]   r   r   )r   r   rC  s   &ffr7   zero_numel_check_dimsrB
  D  sU    yyA~1H!!r	Y	

 	IIcNaU	
r9   c                    a  Ve(   \        W!P                  4       4      p\        WS 4       R # \        P                  ! VP                  4       ^ 8g  V 3R l4       R # )Nc                     < S  R 2# )z@: Expected reduction dim to be specified for input.numel() == 0.r3   r  s   r7   rl   %check_argmax_argmin.<locals>.<lambda>Y  s    tf\]r9   )r   r   rB
  r]   ro   r   )r   r   r   s   f&&r7   check_argmax_argminrF
  R  s?    
S((*-d.JJLA]	
r9   c                     \        R W4       \        P                  ! V P                  Ve   V3MR4      p\	        WV4      pV P                  V\        P                  R7      # )argmaxNr   )rF
  rV   r  r   r  r   r]   r   )r   r   r  r9  r   s   &&&  r7   argmax_argmin_metarI
  ]  sM    $,

coSF4PD$T9E>>%u{{>33r9   c                 ~    V\         P                  8X  d   \         P                  p\         P                  ! RWW4R 7      # )r#  r3   )r]   jaggedr<  r   )r  re   r   r   r   s   &&&&&r7   scalar_tensorrL
  e  s1    
 ;;
%v r9   c                    \        W P                  4       R R7      pV P                  4       ^ 8X  d   ^MV P                  V4      p\        P                  ! V^ 8  4       \        P                  ! W8*  R 4       \        V P                  4      p\        V4      ^ 8  d   WV&   V P                  V4      V P                  V\        P                  R7      3# )Tr  c                      R # )zk not in range for dimensionr3   r3   r9   r7   rl   topk_meta.<locals>.<lambda>w  s    )Gr9   r   )
r   r   r   r]   ro   r   r   r   r   r   )r   r  r   largestr   	sliceSizetopKSizes   &&&&&  r7   	topk_metarT
  q  s     hhjd
;CXXZ1_$))C.I	LLa	LL!GHDJJH
8}q>>(#T^^HEKK^%PPPr9   c                     Vf   Vf   \        R4      hVP                  4       pV P                  4       p	\        P                  ! VV	P                  V	P
                  V	P                  R7      # )Nz;segment_reduce(): Either lengths or offsets must be defined)re   r   r   )r   r	  r]   r   re   r   r   )
r  r  rE  r  r  r  r  r  data_contiggrad_contigs
   &&&&&&&&  r7   meta__segment_reduce_backwardrX
    sh    
 7?I
 	
 //#K//#K!!!!	 r9   c                   a ^ RI Hp \        SV P                  4       RR7      oV P                  4       ^ 8  d   V P	                  S4      M^p\
        P                  ! V! V^8  W8*  4      V3R l4       \        V P                  RS V P                  S^,           R ,           4      pV'       d(   V P                  4       ^ 8  d   VP                  S^4       V P                  V4      V P                  V\
        P                  R7      3# )r   )r,  TrN
  c                     < R S  2# )z9kthvalue(): selected number k out of range for dimension r3   r  s   r7   rl   kthvalue_meta.<locals>.<lambda>  s    KC5Qr9   Nr   )ro  r,  r   r   r   r]   ro   r   r   rz  r   r   )r   r  r   r  r,  dimSizer   s   &&f&   r7   kthvalue_metar]
    s     >
dhhjd
;C $
QdiinAG	LLQ%Q
 DS!DJJsQwy$99:E488:>S!>>% $..ekk."JJJr9   c                    V e   T MTp\         P                  ! VP                  4       ^8H  R 4       VP                  4       pV e*   \         P                  ! V P                  4       V8H  R 4       Ve*   \         P                  ! VP                  4       V8H  R 4       \         P                  ! VP                  4       V8H  R 4       \         P                  ! VP                  4       V8H  R 4       \         P                  ! VP                  4       ^8H  R 4       \         P                  ! VP	                  4       V^ ,          V^,          ,          ^,          8H  R 4       R # )Nc                      R #  r3   r3   r9   r7   rl   (checkLSTMBackwardSizes.<locals>.<lambda>  s    "r9   c                      R # r`
  r3   r3   r9   r7   rl   rb
        r9   c                      R # r`
  r3   r3   r9   r7   rl   rb
    rd
  r9   c                      R # r`
  r3   r3   r9   r7   rl   rb
        r9   c                      R # r`
  r3   r3   r9   r7   rl   rb
    rg
  r9   c                      R # r`
  r3   r3   r9   r7   rl   rb
    s    rr9   c                      R # r`
  r3   r3   r9   r7   rl   rb
    s    Rr9   )r]   ro   r   r   r   )grad_hygrad_cyr
  r
  r
  defined_gradexp_sizes   &&&&&  r7   checkLSTMBackwardSizesro
    s    %17wL	LL!!#q(*5  "HW\\^x/<W\\^x/<	LLh&
3	LLh&
3	LLA%z2	LL"hqkHQK&?!&CCZPr9   c                     V f   Vf   R# \        WW#V4       \        P                  ! V\        R7      p\        P                  ! V\        R7      pV'       d   VP	                  ^ RR7      MR pWgV3# )Nr   F)r  NNN)ro
  r]   r   legacy_contiguous_memory_formatr  )	rk
  rl
  r
  r
  r
  has_bias
grad_gatesgrad_cxr  s	   &&&&&&   r7   #_thnn_fused_lstm_cell_backward_implrv
    si    7?7RY?!!!@J r1PQG4<
q%0$I	))r9   c                 P   R pR pR pV^ ,          '       d    VP                  V P                  4       4      pV^,          '       g   V^,          '       dR   VP                  VP                  R4      V P                  R4      34      pVP                  VP                  R4      4      pWEV3# r  r  )r  r  r  r  r  grad_weightr  s   &&&&   r7   linear_backwardry
    s    JKI1~~!++FKKM:
1~~Q",,l.?.?.CV[[QS_-UV **<+<+<R+@A	Y//r9   c                   a a \        S P                  4      ^8  d&   S P                  R,          W,          ,          ^ 8X  g   \        RS P                   RV 24      hR oVV 3R lpS P                  R,          W,          ,          pS P                  R,          V,          pS P                  R	,          V,          p. S P                  RR OVNVNVN5pS P                  V4      pVP	                  V! 4       R7      pV# )
rA   z'Invalid input shape for pixel_shuffle: z with upscale_factor = c                 b    \         P                  P                  V 4      \         P                  8H  # rH   re  rh  s   &r7   rj  ,meta_pixel_shuffle.<locals>.is_channels_last  s$    ""88=ATATTTr9   c                  \  < S ! S4      '       d1   \        S4      R 8X  d   \        P                  # \        P                  # SP	                  \        P                  R7      '       d   \        P                  # SP	                  \        P
                  R7      '       d   \        P
                  # R# )r   r   N)r   r]   r   rg  r  preserve_format)rj  r   s   r7   rs  .meta_pixel_shuffle.<locals>.pick_memory_format  s    D!!4 F*...***e.E.EFF***e.C.CDD((( Er9   Nr   r  r  r   )r   r   r   r   rr  )	r   upscale_factorrs  r  HrWrr   r   rj  s	   f&      @r7   meta_pixel_shuffler
    s     	DJJ!

2.2Q RVW W5djj\AXYgXhi
 	
U	) 	

2>:;A	B.	(B	B.	(B-$**Sb/-1-b-"-I
..
#C
&&13&
4CJr9   c                 X   V P                  V P                  4      pVP                  VP                  4      pVP                  VP                  4      pVP                  VP                  4      pVP                  VP                  4      pVP                  VP                  4      pVVVVVVV3# rH   r  )r   weight0weight1weight2weight3r8
  cx_tmpr  hy_cy_grad_output_r_optgrad_hy_r_optgrad_cy_r_optr   r  r
  r 
  r:
  r#
  r$
  r?  r!
  r
  diff_xdiff_hxdiff_cxdiff_w1diff_w2diff_bs   &&&&&&&&&&&&&&&&&&&&&&&      r7   mkldnn_rnn_layer_backwardr
    s    4 __U[[)FmmCII&Gv||,G.G.Gw}}-F7GVVWgEEr9   	out_int32rightc                    \         P                  ! T V'       d   \         P                  M\         P                  \         P                  R 7      # )re   r   )r]   r   r  r   r   r   
boundariesr
  r
  s   &&$$r7   meta_bucketizer
    s2     &ekkEKK-- r9   c                H    V ^8  d   QhR\         R\        R\        R\        /# )rA   r   r
  r
  r
  )r   r   r   )rC   s   "r7   rD   rD   !  s0     
 



 	

 
r9   c                ~    TP                  RV'       d   \        P                  R 7      # \        P                  R 7      # )r   r3   )r   r]   r  r   r
  s   &&$$r7   meta_bucketize_scalarr
     s>     
&ekk    ,1KK    r9   c                   a aaaa R o\        S 4      R8X  d*   \        P                  ! S P                  4       V 3R l4       \        S 4      R8X  d-   S P                  4       '       d   \        P
                  ! R4       \        P                  ! \        S\        4      VV3R l4       \        P                  ! S^ 8  VV3R l4       \        P                  ! \        S\        4      VV3R l4       \        P                  ! \        S\        4      VV3R l4       \        P                  ! SS8  V3R	 l4       \        P                  ! SS P                  S P                  R
7      # )zhistc()r   c                  $   < R S P                    R2# )z%"histogram_cpu" not implemented for 'r  r   r  s   r7   rl   meta_histc.<locals>.<lambda>5  s    =ekk]!Lr9   r   z%_histc_cuda with floating point inputc                  $   < S R \        S 4       2# )z#: argument 'bins' must be int, not r  binsrC  s   r7   rl   r
  ;  s    7)>tDzlKr9   c                     < S R S  2# )z: bins must be > 0, but got r3   r
  s   r7   rl   r
  =  s    gY.J4&#Qr9   c                  $   < S  R \        S4       2# )z%: argument 'min' must be Number, not r  )rC  r  s   r7   rl   r
  @      7)@cLr9   c                  $   < S  R \        S4       2# )z%: argument 'max' must be Number, not r  )rC  r  s   r7   rl   r
  D  r
  r9   c                     < S  R 2# )z: max must be larger than minr3   )rC  s   r7   rl   r
  F  s    y0M%Nr9   r   )r   r]   ro   r  rV   r  ry   r   r   r   r   re   )r   r
  r  r  rC  s   ffff@r7   
meta_histcr
  .  s     G5U"##%L	
 5V#(?(?(A(A%%&MN	LL4!K 
LLQR	LL3L 
LL3L 
LLNO;;tELLDDr9   c                   a  \        S P                  4       V^R7      p\        P                  ! S P	                  4       ^ 8g  ;'       g_    \
        ;QJ d1    R S P                  4       R,           4       F  '       d   K   RM&	  RM"! R S P                  4       R,           4       4      V 3R l4       S P                  V4      P                  \        P                  ! S 4      R7      # )rA   r	  c              3   *   "   T F	  q^ 8  x  K  	  R# 5ir	  r3   )r{   r   s   & r7   r}   ,meta_upsample_bimode2d_aa.<locals>.<genexpr>\  s     !H7Gt(7Gr	  r	  FTc                  *   < R S P                  4        2# r	  r   r  s   r7   rl   +meta_upsample_bimode2d_aa.<locals>.<lambda>]  r	  r9   r   )
r	  r   r]   ro   r   rZ  r   rr  rV   r   )r   rB  r  r	  r	  r	  s   f&&&& r7   meta_upsample_bimode2d_aar
  J  s     -

kA 
LLHHcc!Huzz|B7G!Hccc!Huzz|B7G!HH_ ??+,//11%8 0  r9   c                 p  a aa \        W!^R7      o\        P                  ! S P                  ^8H  V 3R l4       \	        ^4       F;  o\        P                  ! S P
                  S,          SS,          8H  VV V3R l4       K=  	  S P                  V4      P                  \        P                  ! S 4      R7      # )rA   r	  c                  "   < R S P                    2# r	  rv  rn  s   r7   rl   4meta_upsample_bimode2d_aa_backward.<locals>.<lambda>w  r	  r9   c            
      L   < R S RS S,           RS RSP                  S4       2# )zD
Expected grad_output to have the same shape as output; output.size(r	  z
but got grad_output_size(r   r	  s   r7   rl   r
  |  s>     DDE3dK[\]K^J_ `D!1!1!!4 59r9   r   )
r	  r]   ro   r   r   r   r   rr  rV   r   )r  rB  r  r  r	  r	  r	  r   s   f&&&&&@@r7   "meta_upsample_bimode2d_aa_backwardr
  d  s     -! 
LLAk 1Xa $4Q$779	
    ,//11+> 0  r9   c                 Z   \         P                  ! VP                  4       ^8H  R 4       \         P                  ! VP                  4       ^8H  R 4       \         P                  ! VP                  P                  R 4       \         P                  ! VP                  P                  R 4       R# )r2   c                      R # )z%found_inf must be a 1-element tensor.r3   r3   r9   r7   rl   <_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>      (Or9   c                      R # )z%inv_scale must be a 1-element tensor.r3   r3   r9   r7   rl   r
    r
  r9   c                      R # )z!found_inf must be a float tensor.r3   r3   r9   r7   rl   r
        3r9   c                      R # )z!inv_scale must be a float tensor.r3   r3   r9   r7   rl   r
    r
  r9   N)r]   ro   r   re   r  )r   r  	inv_scales   &&&r7   *_amp_foreach_non_finite_check_and_unscale_r
    s|    	LLQ O 
LLQ O 
LL))3 
LL))3r9   c                 .    \         P                  ! V 4      # rH   r  )r   nanposinfneginfs   &&&&r7   
nan_to_numr
    r  r9   c                     V P                   \        P                  \        P                  \        P                  \        P
                  09   d   \        R V P                    R24      hV P                  p\        W4      p\        W#4      pW8X  d   V # \        V P                  4       4      p\        V P                  4       4      pWR,          WQ,          uWQ&   WR&   WB,          WA,          uWA&   WB&   V P                  WE4       V # )z>torch.transpose_: in-place transposition is not supported for z layout)r   r]   r=  
sparse_cscr>  
sparse_bscr   r   r   r   r   r   r   )r   dim0r  ndimsr   r   s   &&&   r7   r/  r/    s    {{	  LT[[MY`a
 	
 IIE$&D$&D|		D$++- F!'v|FL&,!ZDJ
T"Kr9   c                 <   V P                   pV P                  '       dB   V P                  4       pV P                  4       pV^8:  d   V^ 8X  g   \	        RV RV R24      hM$V P                  4       ^8  d   \	        RV R24      h\        T ^ V^8  d   ^ 4      # ^4      # )rA   zEt_ expects a tensor with <= 2 sparse and 0 dense dimensions, but got z sparse and z dense dimensionsz6t_ expects a tensor with <= 2 dimensions, but self is r  )r   r  r  r  r   r   r/  )r   r
  r  r  s   &   r7   t_r
    s    IIE~~~__&
NN$	aIN %,l9+=NP  %3 88:> HqQ  dAEAIq55155r9   sidesorterc                  a aa \         P                  ! \        S P                  4      ^8*  ;'       g     S P                  RR SP                  RR 8H  VV 3R l4       \         P                  ! SRJ ;'       g    S P                  SP                  8H  V V3R l4       \         P                  ! VR8g  ;'       g    V'       * R 4       V'       d   \         P                  M\         P
                  p\        S\         P                  4      '       d(   \         P                  ! SV\         P                  R7      # \         P                  ! R	VS P                  R7      # )
r2   Nc                  `   < R \        SP                  4       R\        S P                  4       2# )ztorch.searchsorted(): boundaries tensor should be 1 dimension or the first N-1 dimensions of boundaries tensor and input value tensor must match, but we got boundaries tensor z and input value tensor rX  )r   sorted_sequences   r7   rl   #meta_searchsorted.<locals>.<lambda>  s3    3378M8M3N2O P""&tzz"2!35r9   c                  p   < R \        S P                  4       RSe   \        SP                  4       2# .  2# )z[torch.searchsorted(): boundary and sorter must have the same size, but got boundary tensor z and got sorter tensor rX  )r
  r
  s   r7   rl   r
    sB    ##'(=(=#>"??V%+%7tFLL!@B=?@Br9   r   c                      R # )zetorch.searchsorted(): side and right can't be set to opposites, got side of left while right was Truer3   r3   r9   r7   rl   r
    s     $r9   r
  r  r   r3   )r]   ro   r   r   r  r   ry   r   r   r   r   r   )r
  r   r
  r
  r
  r
  re   s   ff$$$d r7   meta_searchsortedr
    s    
LLO!!"a' 	9 	9  "%CR8	
	 
LL$??///6<<?	
 
LL##e)	$ %EKK%++E$%%U-D-D
 	
 {{2U?3I3IJJr9   c                    a  \         P                  ! S \         P                  \         P                  \         P                  39  V 3R  l4       R# )c                     < R S  2# )z/Unsupported input type encountered for isin(): r3   r   s   r7   rl   3_check_for_unsupported_isin_dtype.<locals>.<lambda>   s    A%Ir9   N)r]   ro   r   
complex128	complex64r   s   fr7   !_check_for_unsupported_isin_dtyper
     s/    	LLejj%"2"2EOODDIr9   c                 H    V P                  W P                  R4      34      pV# r  r  )r  r   num_weightsr  r  rx
  s   &&&&& r7   meta_embedding_dense_backwardr
     s(     ''6F6Fr6J(KLKr9   c                 v    V	'       d   \         P                  V VVVVVVVV
V4
      # \        V VVVVVVVV
V4
      # rH   )r-   _embedding_bag_sparse_backward!meta_embedding_bag_dense_backward)r  r   r  r  r  maximum_indicesr
  r  r  r  r  r  s   &&&&&&&&&&&&r7   meta_embedding_bag_backwardr
     se     22
 	
 1
 	
r9   c
                 Z  a  \         P                  ! S P                  \         P                  \         P                  \         P
                  \         P                  39   V 3R  l4       V\        8X  d   \         P                  ! VRJ4       S P                  VS P                  ^4      34      p
V
# )c                  "   < R S P                    2# )z$Unsupported input type encountered: r   )r  s   r7   rl   3meta_embedding_bag_dense_backward.<locals>.<lambda>Q   s    6tzzlCr9   N)
r]   ro   re   r  r  r  float64r  r   r   )r  r   r  r  r
  r
  r  r  r  r  index_grad_weights   f&&&&&&&&& r7   r
  r
  B   sv     
LL

u}}ennemmU]]SSC x_D01TYYq\'BCr9   c                    V P                  ^4      p\        P                  ! V\        8H  R 4       \        P                  ! V P	                  4       ^8H  4       \        P                  ! VP	                  4       ^8H  4       VP                  ^ 4      p\        P                  ! VP	                  4       ^8H  4       \        P                  ! VP                  ^4      V8H  4       V P                  V34      p	V	# )r2   c                      R # )zHembedding_bag_backward: per_sample_weights only supported for mode='sum'r3   r3   r9   r7   rl   @meta_embedding_bag_per_sample_weights_backward.<locals>.<lambda>f   r  r9   )r   r]   ro   r  r   r   )
r  r  r   r  r  r  r  embedding_featuresr	  r  s
   &&&&&&&   r7   .meta_embedding_bag_per_sample_weights_backwardr
  Y   s     1	LLZ 
LLq!	LL!#$,,q/K	LL"#	LLQ#556^^[N+FMr9   assume_uniqueinvertc                   \         P                  ! \        V \        4      ;'       g    \        V\        4      R  4       \        V \        4      '       g"   \         P                  ! WP
                  R7      p \        V\        4      '       g"   \         P                  ! WP
                  R7      p\        V P                  4       \        VP                  4       \         P                  ! V \         P                  R7      # )c                      R # )z<At least one of elements and test_elements must be a Tensor.r3   r3   r9   r7   rl   meta_isin.<locals>.<lambda>v   r  r9   rN  r   )
r]   ro   ry   r   r{  r   r
  re   r   r   )elementstest_elementsr
  r
  s   &&$$r7   	meta_isinr
  q   s     
LL8V$II
=&(IN h''<<1E1EFmV,,]??K%hnn5%m&9&9:HEJJ77r9   c                <    V ^8  d   QhR\         R\        R\        /# )rA   r  r   rB   )r   r   )rC   s   "r7   rD   rD      s!     6 6c 6 6F 6r9   c                     \         P                  ! V ^ 8  R 4       \        V\        P                  R7      w  r#\         P
                  ! WR7      # )r   c                      R # )z,polygamma(n, x) does not support negative n.r3   r3   r9   r7   rl    meta_polygamma.<locals>.<lambda>   s    !Or9   r  r   )r]   ro   r   r   r  r   )r  r   rY   rZ   s   &&  r7   meta_polygammar
     sB     
LLaOP(;HHOA D55r9   c                $    V ^8  d   QhR\         /# )rA   r   r_  )rC   s   "r7   rD   rD      s     I I& Ir9   c                     \        R 4      h)z.Tensor.item() cannot be called on meta tensors)rq  r   s   &r7   meta_local_scalar_denser
     s    
G
HHr9   c                0    V ^8  d   QhR\         R\         /# rA   r   rB   r_  )rC   s   "r7   rD   rD      s     " "v "& "r9   c                 .    \         P                  ! V 4      # rH   r  r   s   &r7   silur     r  r9   c                0    V ^8  d   QhR\         R\         /# r
  r_  )rC   s   "r7   rD   rD      s     6 6& 6V 6r9   c                 j    \        V \        P                  R 7      w  r\        P                  ! WR7      # r  )r   r   r  r]   r   )r   rY   rZ   s   &  r7   sigmoidr     s/     );HHOA D55r9   c                    V P                  4       ^8H  pVP                  4       ^8H  pV'       d   V'       d5   VP                  ^ 4      V P                  ^ 4      VP                  ^4      .pEM+\        P                  ! VP                  ^ 4      VP                  ^ 4      8H  R 4       V P                  ^ 4      VP                  R4      .pMV'       d]   \        P                  ! VP                  ^ 4      V P                  ^ 4      8H  R 4       V P                  ^4      VP                  ^4      .pMk\        P                  ! V P                  ^ 4      VP                  ^ 4      8H  R 4       V P                  ^ 4      V P                  ^4      VP                  R4      .pT;'       g    V P                  p\        P
                  P                  '       d{   ^VP                  ,          pVR,          V,           ^,
          V,          V,          pWE8X  d   V^,          V,          V^.p	MV^.p	\        P                  ! WiW0P                  R7      p
V
# \        P                  ! WcV P                  R7      p
V
# )rA   c                      R # z matrix batch sizes have to matchr3   r3   r9   r7   rl   2_create_grouped_mm_output_tensor.<locals>.<lambda>       6Xr9   c                      R # r  r3   r3   r9   r7   rl   r     r	  r9   c                      R # )zbatched dimension has to matchr3   r3   r9   r7   rl   r     s    6Vr9   r  r   )r   r   r]   ro   re   r]  r   itemsizer  r   r   )r  r  offsr  
mat1_is_2d
mat2_is_2dr+
  	alignmentsize_paddedr	  r   s   &&&&       r7    _create_grouped_mm_output_tensorr     s   qJqJ		!diilDIIaLAHLL		!		!,.X 		!diim4HLL		!		!,.X 		!diil3H LL		!		!,.V 		!diilDIIbMBH''TZZI}}),,,	|i/!3	AIM#"1+3[!DJ%q)J!!	++

 J kk(DKKHJr9   c                (   V ^8  d   QhR\         R\         R\        P                   R,          R\        P                   R,          R\         R,          R\         R,          R\        P                   R,          R	\        P                  R,          R
\        /	# )rA   mat_amat_br  Nr  r  r  r  r  r  )r   r]   re   r   )rC   s   "r7   rD   rD      s     VK VKVKVK \\D VK \\D 	VK
 4-VK 4-VK ,,%VK {{T!VK VKr9   c	                 	  a aaaaa \         P                  ! SR J SR J 8H  R 4       SR J;'       d    SR Jp	V	'       d   \         P                  p
\         P                  P                  '       dc   \         P
                  P                  4       '       d?   R\         P
                  P                  ^ 4      P                  9   d   \         P                  p
\         P                  ! S P                  V
8H  ;'       d    SP                  V
8H  V V3R l4       M\\         P                  ! S P                  \         P                  8H  ;'       d    SP                  \         P                  8H  V V3R l4       \         P                  ! S P                  4       R9   ;'       d    SP                  4       R9   V V3R l4       S P                  4       ^8H  pSP                  4       ^8H  pV'       d	   V'       g:   \         P                  ! S P                  R4      SP                  R4      8H  R 4       V	'       dI   R pR p\         P                  ! V! S 4      V 3R	 l4       \         P                  ! V! S4      V3R
 l4       R pV! RS 4       V! RS4       SEeQ   SEeL   \         P                  ! SP                  \         P                  8H  ;'       d&    SP                  \         P                  8H  ;'       gC    SP                  \         P                  8H  ;'       d    SP                  \         P                  8H  VV3R l4       SP                  \         P                  8H  ;'       d    SP                  \         P                  8H  oRV3R llpSe$   V'       d   V'       d   SP                   ^ ,          M^pV! RSS ^ V4       V! RSS^V4       \         P                  ! VR J R 4       V'       g	   V'       d   \         P                  ! SR JV V3R l4       Sec   \         P                  ! SP                  4       ^8H  V3R l4       \         P                  ! SP                  \         P"                  8H  V3R l4       M\         P                  ! SR J R 4       \         P                  ! VR J R 4       \         P                  ! VR J ;'       g    V\         P                  8H  R 4       \%        S SSV4      # )Nc                      R # )z,Either both scale factors are given, or noner3   r3   r9   r7   rl   )_meta_grouped_mm_common.<locals>.<lambda>   s    >r9   gfx94c                  >   < R S P                    RSP                    R2# )z5Expected inputs of E4M3 FP8 type but got mat_a.dtype= and mat_b.dtype=r  r   r  r  s   r7   rl   r     s#    KEKK=Xijojujuivvwxr9   c                  >   < R S P                    RSP                    R2# )z1Expected inputs of BF16 type but got mat_a.dtype=r  r  r   r  s   r7   rl   r     s#    G}Tefkfqfqerrstr9   c                  L   < R S P                  4        RSP                  4        2# )z3Multiplicands must be 2D or 3D but got mat_a.dim()=z and mat_b.dim()=r  r  s   r7   rl   r     s%    Eeiik]Rcdidmdmdocpqr9   c                      R # )z3contraction dimension of mat_a and mat_b must matchr3   r3   r9   r7   rl   r     r  r9   c                 b    V P                  4       pVR,          ^8  ;'       d    VR,          ^8H  # rA   r  r   r  mat
mat_strides   & r7   r	  -_meta_grouped_mm_common.<locals>.is_row_major!  s-    Jb>A%==*R.A*==r9   c                 b    V P                  4       pVR,          ^8H  ;'       d    VR,          ^8  # r!  r  r"  s   & r7   r	  -_meta_grouped_mm_common.<locals>.is_col_major!  s-    Jb>Q&==:b>A+==r9   c                  0   < R S P                  4       RR  2# )zNExpected mat_a tensor to be row major in the last two dimensions, got strides Nr  r  )r  s   r7   rl   r  !  s!    dejeqeqestvtwexdyzr9   c                  0   < R S P                  4       RR  2# )zQExpected mat_b tensor to be column major in the last two dimensions, got strides Nr  r  )r  s   r7   rl   r  !  s!    ghmhththvwywzh{g|}r9   c                   a aaa SP                  4       ^,
          o^SP                  4       ,          pSP                  4       oSS^,
          ,          ^8X  da   SS,          \        ^SP                  S^,
          ,          4      8  d1   \
        P                  ! SS,          V,          ^ 8H  VV V3R l4       R# SS,          ^8X  dh   SS^,
          ,          \        ^SP                  S,          4      8  d8   \
        P                  ! SS^,
          ,          V,          ^ 8H  VV V3R l4       R# \
        P                  ! RVV3R l4       R# )r2   c                  *   < R S RS  RSS ,           R2# rc  z stride along z% dim to be multiple of 16 bytes, got r  r3   end_dimmat_namer$  s   r7   rl   F_meta_grouped_mm_common.<locals>.check_valid_strides.<locals>.<lambda>!  s'    )H:^G9Dijtu|j}i~~  Ar9   c                  F   < R S RS ^,
           RSS ^,
          ,           R2# r,  r3   r-  s   r7   rl   r0  %!  sI    )H:^GaK=Hmnx  zA  DE  zE  oF  nG  GH  Ir9   Fc                  *   < R S RS P                    R2# )zInvalid strides/sizes, got z for strides and z for sizes.r8  r"  s   r7   rl   r0  *!  s    5j\ARSVS\S\R]]hir9   N)r   element_sizer   r  r   r]   ro   )r/  r#  r  r.  r$  s   ff @@r7   check_valid_strides4_meta_grouped_mm_common.<locals>.check_valid_strides!  s    '')a-#**,,	ZZ\
gk"a'Jw,?3syy1%D
 -
 LL7#i/14 A  A%*Wq[*ASsyy!F
 +
 LL7Q;')3q8 I
 LLir9   r  r  c                  >   < R S P                    RSP                    R2# )zhFor FP8 scales must both be float32, or for MXFP8 both scales must be float8_e8m0fnu. Got scale_a.dtype=z and scale_b.dtype=r  r   r1	  s   r7   rl   r  7!  sR    ~  @G  @M  @M  N  Na  bi  bo  bo  ap  pq  rr9   c                 x  <a aaaaaa	a
 SP                  4       ^8X  d   \        P                  ! SP                  4       V 3R l4       S'       d?   \        P                  ! SP                  4       SP                  4       8H  VVV 3R l4       R# \        P                  ! SP                  4       ^8H  VV 3R l4       \        P                  ! SP                  ^ ,          SP                  S,          S,          8H  VVVV V3R l4       R# \        P                  ! SP                  R4      ^8H  V 3R l4       \        P                  ! SP                  ^ ,          SP                  ^ ,          8H  VVV 3R l4       S'       d   \        P                  ! SP                  SP                  ^,
          8H  VVV 3R l4       SP                  w  orV^ p\        WW,          ^4      o	\        V^4      o
\        P                  ! SP                  ^ ,          S8H  ;'       d    SP                  ^,          S	S
,          8H  VV	V
VV3R l4       R# \        P                  ! SP                  4       ^8H  VV 3R	 l4       \        P                  ! SP                  ^,          SP                  ^S,           ,          8H  VVV V3R
 l4       R# )rA   c                     < R S  R2# )rc  z to be contiguous.r3   
scale_names   r7   rl   >_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>B!  s    i
|3EFr9   c                  B   < R S RS P                    RSP                    2# )zKFor MXFP8, scale must have same number of dimensions as target tensor, but  has mat.ndim= and scale.ndim=rv  r#  r!  r:  s   r7   rl   r;  K!  sX    "mnxmy  zH  IL  IQ  IQ  HR  Rb  ch  cm  cm  bn  !or9   c                  2   < R S RS P                  4        R2# )rc  z to be 1D tensor, but got 	D tensor.r  r!  r:  s   r7   rl   r;  P!  !    )J<7QRWR[R[R]Q^^g hr9   c                  n   < R S RS P                   S,          S,           RSP                   ^ ,           R2# )rc  z	 to have r,	  z
 elements.r8  )r#  r!  scale_multiplierr:  
scaled_dims   r7   rl   r;  T!  sU    )J<y:AVYiAi@jjyz  {F  {F  GH  {I  zJ  JT  !Ur9   c                     < R S  R2# )rc  z( to be contiguous in the last dimension.r3   r9  s   r7   rl   r;  Y!  s    i
|3[\r9   c                  `   < R S RS P                   ^ ,           RSP                   ^ ,           R2# )rc  z batch dimension to be , got r  r8  r?  s   r7   rl   r;  ]!  s4    i
|3J399UV<.X^_d_j_jkl_m^nnopr9   c                  B   < R S RS P                    RSP                    2# )z0For MXFP8, 3d tensor should have 2d scales, but r=  r>  rv  r?  s   r7   rl   r;  d!  sD    "RS]R^^lmpmumulv  wG  HM  HR  HR  GS  !Tr9   c                  V   < R SP                    RS  RSS,           RSP                    2# )zFor MXFP8, expected mat.shape=z to have scale shape of (,z), but got r8  )G	blocked_K	blocked_Nr#  r!  s   r7   rl   r;  n!  sO    "@Kdefdgghiru~i~h  @K  LQ  LW  LW  KX  !Yr9   c                  2   < R S RS P                  4        R2# )rc  z to be 2D tensor, but got rA  r  rB  s   r7   rl   r;  s!  rC  r9   c                  n   < R S RS P                   ^S,           ,           RSP                   ^,           R2# )rc  z non-batch dimension to be rI  r  r8  )r#  r!  r:  rF  s   r7   rl   r;  w!  sR    )J<7RSVS\S\]^ak]kSlRmmstytt  AB  uC  tD  DE  !Fr9   Nr   )r   r]   ro   r  r   r   r   r>   )r:  r!  r#  rF  rE  rW  r4  rd  rM  rN  rO  is_mxfp8s   fffff   @@@r7   check_scale,_meta_grouped_mm_common.<locals>.check_scale>!  s   wwyA~'')F LL		swwy0 o
 LL		q(h LLA#))J*?BR*RR U  U
 LL$)\ KKNciil2p LL

chhl2 T "iiGAq!#J ( ;I (C 0ILLA!+WWA)iBW0W Y  Y
 LL		q(h LLA#))A
N*CC Fr9   r  r  c                      R # )z:Scale result tensor provided, but it is not supported yet.r3   r3   r9   r7   rl   r  !  rv   r9   c                  N   < R S P                  4        RSP                  4        R2# )z/Offsets tensor not provided, but is needed for zD/zD multiplicand layouts.r  r  s   r7   rl   r  !  s(    Eeiik]RTUZU^U^U`Taaxyr9   c                  ,   < R S P                  4        R2# )z.Offsets tensor must be 1D, but got offs.dim()=r  r  r  s   r7   rl   r  !  s    HTUVr9   c                  $   < R S P                    R2# )z7Offsets tensor must be integer (int32) tensor, but got r  r   rX  s   r7   rl   r  !  s    QRVR\R\Q]]^_r9   c                      R # )zJOffsets tensor provided, but is not needed for 3D/3D multiplicand layouts.r3   r3   r9   r7   rl   r  !  s    `r9   c                      R # )z2Bias tensor provided, but it is not supported yet.r3   r3   r9   r7   rl   r  !  s    Dr9   c                      R # )z4If output dtype provided, it must be torch.bfloat16.r3   r3   r9   r7   rl   r  !  s    Fr9   )rA   r  r   r  r1   )r]   ro   r  r]  r^  r   r  get_device_propertiesgcnArchNamer  re   r  r   r   r  r>	  r   r  r  )r  r  r  r  r  r  r  r  r  scaled	fp8_dtypemat_a_is_2dmat_b_is_2dr	  r	  r4  rS  rE  rR  s   fffff&&&&         @r7   _meta_grouped_mm_commonrc     s    
LL	Dgo.> D 88WD%8F ''	MM

''))5::;;A>JJJ--IKK9$AA	)Ax	

 	KK5>>)KKekkU^^.Kt	

 
LL		v77%))+"7q
 ))+"K))+"KkJJrNejjn,I	

 	>	> 	z	
 	}	

0 ''w2]]emm+NN0N  !5!55 : :MMU%9%99 r	
 MMU111 6 6!5!55 	
:	z "-++DJJqMST 	 	Iwq2BCIwq2BCD P	

 ky	
 LL
aV LL

ekk)_
 	DL`	

 
LLD
 
LLT88Y%..8F
 ,E5$	JJr9   c                    V ^8  d   QhR\         R\         R\         R,          R\         R,          R\        P                  R,          R\         /# )rA   r  r  r  Nr  r  rB   r  )rC   s   "r7   rD   rD   !  sU       4- 4-	
 {{T! r9   c                 (    \        V VR R VVR VR7      # )N)r  r  r  r  r  r  )rc  )r  r  r  r  r  s   &&&&&r7   meta_grouped_mmrf  !  s)     #	 	r9   c                \   V ^8  d   QhR\         P                  R\         P                  R\         P                  R\         P                  R\         P                  R,          R\         P                  R,          R\         P                  R,          R	\         P                  R,          R
\        /	# )rA   r  r  r  r  r  Nr  r  r  r  r  )rC   s   "r7   rD   rD   !  s      <<<< \\ \\	
 ,,
 ,,
 ,,% {{T! r9   c	                 ^    T;'       g    \         P                  p\        V VVVVVVVVR 7	      # ))r  r  r  r  r  r  r  )r]   r  rc  )	r  r  r  r  r  r  r  r  r  s	   &&&&&&&&&r7   meta_scaled_grouped_mmri  !  s=     ++U^^I"!%
 
r9   c                 T   \        V4      \        R 4      8X  d3   V  F,  p\        P                  ! VP                  4       ^ 8  R 4       K.  	  . pV  FT  pVe   TMVP                  pVP
                  '       d   \        V4      pVP                  VP                  RVR7      4       KV  	  V# )infc                      R # )z:_foreach_norm cannot compute infinity norm on empty tensorr3   r3   r9   r7   rl   #meta_foreach_norm.<locals>.<lambda>!  s    Tr9   r   r3   )	ra   r]   ro   r   re   r   r   r[  r   )tensorsordre   r,  resultsr  s   &&&   r7   meta_foreach_normrq  !  s    SzU5\!ALL	AT 
 G".EAGG	0;Iq{{2Y{78	 
 Nr9   c                H    V ^8  d   QhR\         R\        R\        R\         /# )rA   r<   r   half_to_floatrB   r   r   r   )rC   s   "r7   rD   rD   !  s(      v C   r9   c                 n   V'       dI   V P                   \        P                  \        P                  39  d   \	        R V P                    R24      h\
        P                  ! V \
        P                  P                  R7      w  r4V'       g   TMTp\        P                  ! W\        P                  R7      pV# )z%half_to_float is True but x.dtype is z, expected half or bfloat16r  r
  )re   r]   r_   r  r   rV   r   r   rW   r   r   )r<   r   rs  computation_dtyperZ   r(  s   &&&   r7   softmaxrw  !  s     775::u~~66 7y@[\  ',&>&>	uDDLL'# (5<:KL


1@W@W
XCJr9   c           	        aaaa	a
a \         P                  ! \        S4      ^,          ^ 8H  V3R l4       V P                  o\        S4      o
\        S4      ^,          pS
V,
          o	\         P                  ! S
V8  V
V3R l4       \        ;QJ d    R S 4       F  '       d   K   RM	  RM! R S 4       4      '       d   T p\        S	S
4       F  o^S
S,
          ^,
          ,          oSS,          ^ 8  d;   VP                  SSS,          ) VP                  S,          SS,          ,           4      pSS^,           ,          ^ 8  g   Ky  VP                  S^ VP                  S,          SS^,           ,          ,           4      pK  	  VP                  4       # \        SRS	 4      p\        V4       F  o\        S4      S^,           ^,          ,
          oSS	S,           ,          SS,          ,           SS^,           ,          ,           p\         P                  ! V^ 8  VVV	VV3R l4       VP                  V4       K  	  \         P                  ! VV P                  V P                  V P                  \        V 4      R7      # )	rA   c                      < R \        S 4       2# )z1Length of pad must be even but instead it equals r_  rM  s   r7   rl   '_constant_pad_nd_meta.<locals>.<lambda> "  s    CCH:Nr9   c                  (   < R \        S4       RS  R2# )z`Length of pad should be no more than twice the number of dimensions of the input. Pad length is z while the input has z dimensions.r_  )l_inprN  s   r7   rl   rz  
"  s      225c(;P'r9   c              3   p   "   T F,  p\        V\        P                  4      ;'       d    V^ 8*  x  K.  	  R# 5ir	  )ry   rV   IntWithoutSymInt)r{   r  s   & r7   r}   (_constant_pad_nd_meta.<locals>.<genexpr>"  s,     
IS:a//0;;Q!V;Ss   %66FTNc            	      v   < R SSS ,           ,           RSS,           RSS^,           ,           RSS ,            R2	# )zThe input size z, plus negative padding r   zG resulted in a negative output size, which is invalid. Check dimension z of your input.r3   )r   r  l_diffrN  pad_idxs   r7   rl   rz  #"  sE    ok&1*&=%>>V7|nE#gk"2!3 4117!OMr9   )re   r   r   r   )r]   ro   r   r   rZ  r   narrowr   r   r[  r   re   r   r   r   )r   rN  r  l_padc_input	new_shapenew_dimr   r  r  r|  r  s   &f&    @@@@@r7   _constant_pad_nd_metar  !  s    
LLC1N
 ++KEHMEU]F	LL	 s
IS
Isss
IS
IIIvu%A519q=)G7|a!..G}gmmA&6W&E 7Q;!#!..Aw}}Q/?#gPQkBR/RS & }}[&)*I5\c(q1uk*fqj)CL83w{;KKqLM M	
 	!  ;;kk||))+E2 r9   c                `    V ^8  d   QhR\         R\         R\        R\        R\        R\         /# )rA   r  r   r  r  r  rB   rt  )rC   s   "r7   rD   rD   4"  sD     8 888 8 	8
 8 8r9   c                 n   V P                  4       ^8w  d   \        RV P                  4        R24      hV P                  pVP                  pVP                  ^ 8X  d   V^,          3pM2VP                  ^8X  d   V^ ,          V^,          3pM. VOV^,          N5pV P                  pV P                  WxR7      # )rA   z'weight' must be 2-D, got z-Dr   )r   r   r   r   re   r   )	r  r   r  r  r  weight_shapeindices_shaper   r  s	   &&&&&    r7   	embeddingr  2"  s     zz|q9&**,rJKK<<LMMM||q&21o%7			"1%|A7	5m5\!_5	II77r9   c                t    V ^8  d   QhR\         R\        \         ,          R\        \        ,          R\        /# )rA   r   r  max_lengthspadding_value)r   r   r   ra   )rC   s   "r7   rD   rD   L"  s8     * **&\* c* 	*r9   c                 F   \        V4      ^8w  d   \        R\        V4       R24      h\        V4      ^8w  d   \        R\        V4       R24      hV^ ,          P                  ^ ,          ^,
          pV^ ,          pWE.V P                  R,          O5pV P                  V4      # )r2   z&Only one jagged dim is supported, got z offsetsz max_lengthsr	  )r   r   r   r   )r   r  r  r  r;  r  r  s   &&&&   r7   $meta__jagged_to_padded_dense_forwardr  K"  s     7|q4S\N(K
 	
 ;14S5E4FlS
 	
 	
aAAA,6<<+,LL))r9   c                 B    \        V 4      \        4       R  4       4       pV# )c                 8    \        V \        P                  R 7      # r  r[   r   r  r  s   &r7   _f)_create_unary_float_meta_func.<locals>._fc"  s      =JJ
 	
r9   rQ   r$   funcr  s   & r7   _create_unary_float_meta_funcr  b"  *    4]
  

 Ir9   c                 4   V P                   '       g%   VP                   '       g   VP                   '       d   \        R 4      hV P                  4       ^ 8X  d-   V P                  V P                  4      V P                  ^ 4      3# V P                  ^ 4      pV P                  ^4      pVP                  ^ 4      pV P                  WV4      pV
'       d6   V'       d   V P                  WV4      pVV3# V P                  WW4      p VV3# V P                  ^ 4      pVV3# )zP_native_multi_head_attention fake implementation does not support nested tensors)	is_nestedr  r   r   r   r   )rS  r   r  	embed_dimr  
qkv_weightqkv_biasproj_weight	proj_biasr)  need_weightsaverage_attn_weights	mask_typer;  T
output_dimr  attn_weightss   &&&&&&&&&&&&&     r7    native_multi_head_attention_faker  n"  s      #---5???!^
 	
 {{},eooa.@AA

1A

1A !!!$J__Q:.F !??13L L!!	 !??1=L L!! q)L!!r9   c                 B    \        V 4      \        4       R  4       4       pV# )c                 8    \        W\        P                  R 7      # r  r  r;   s   &&r7   r  *_create_binary_float_meta_func.<locals>._f"  s      !@!M!M
 	
r9   r  r  s   & r7   _create_binary_float_meta_funcr  "  r  r9   c                    a  \        S 4      V 3R  l4       pS P                   R2pW!n        \        \        \        V4      4      ! V4      pV# )c                 `   < S! V .VO5/ VB p\        V P                  VP                  4       V # rH   r  )r   rX   r  r   rJ   s   &*, r7   _fn#_register_inplace_meta.<locals>._fn"  s.    '''

CII6r9   rY   )r   r   rQ   getattrr-   )rJ   r  inplace_names   f  r7   _register_inplace_metar  "  sK    
2Y 
 kk]!$LL
l3
4S
9CJr9   c                 ~  a aa \         P                  ! S P                  SP                  8H  VV 3R  l4       S S.p\        S\        4      '       dV   SP
                  ^ 8w  d4   \         P                  ! S P                  SP                  8H  V V3R l4       VP                  S4       \        VR\        P                  / # )c                  <   < R SP                    RS P                    2# )rQ  z for `end`, but got dtype r   )r   r   s   r7   rl   lerp.<locals>.<lambda>"  s    /%++.HTr9   c                  <   < R S P                    RSP                    2# )rQ  z for `weight`, but got dtype r   )r   r  s   r7   rl   r  "  s    /%++6STZT`T`Sabr9   rS   )
r]   ro   re   ry   r   r   r[  r[   r   rW   )r   r   r  rX   s   fff r7   lerpr  "  s     
LLsyy T 3<D&*%%;;!LLv||+b 	F	=EE r9   r  c                :    \        WV\        P                  R 7      # r  r  r   tensor1tensor2r  s   &&&$r7   addcmulr  "  s     0O0W0W r9   c                    \         P                  ! \        P                  ! VP                  4      ;'       d!    \        P                  ! VP                  4      '       * R  4       \        WV\        P                  R7      # )c                      R# )FInteger division with addcdiv is no longer supported, and in a future )r  zErelease addcdiv will perform a true division of tensor1 and tensor2. z4The historic addcdiv behavior can be implemented as zA(input + value * torch.trunc(tensor1 / tensor2)).to(input.dtype) zfor integer inputs and as z6(input + value * tensor1 / tensor2) for float inputs. z?The future addcdiv behavior is just the latter implementation: z4(input + value * tensor1 / tensor2), for all dtypes.r3   r3   r9   r7   rl   addcdiv.<locals>.<lambda>"  s     	
r9   r  )r]   ro   rV   r  re   r[   r   rW   r  s   &&&$r7   addcdivr  "  se     
LL""7==1 6 6&&w}}5	
		
  0O0W0W r9   c                  ^   / p R	 F+  p\         V,          pV F  pW09  g   K  W#,          W&   K  	  K-  	  V P                  4        EF  w  rE\        V\        P                  P
                  4      '       d   K2  \        V\        4      '       g   \        R\        V4       24      hVP                  \        P                  P                  P                  4      ! V4       \        P                  P                  VP                  4       R4      '       d#   V\         R ,          9   d   \        V R24      hK  VP                   '       d   EK  VP                  4       R
9   d   EK  RVP                  4       9   d   \"        P%                  WE4       EKH  RVP                  4       9   d   \&        P%                  WE4       EKu  RVP                  4       9   d   \(        P%                  WE4       EK  RVP                  4       9   d   \*        P%                  WE4       EK  \,        P%                  WE4       EK  	  R# )r   z$op_overload must be OpOverload, got CompositeImplicitAutogradz is a CompositeImplicitAutograd op, we shouldn't register meta function for it. Instead, we should let the decomposition run and write meta kernels for the base operators.zmkldnn::zmkl::zonednn::zquantized::N)r   post_autogradpre_autograd>   aten::cloneaten::copy_aten::rot90aten::_to_copyaten::empty_stridedaten::constant_pad_ndaten::as_strided_scatter)r   itemsry   r]   _opsHigherOrderOperatorr   r   r   py_impl_CDispatchKeyr/   %_dispatch_has_kernel_for_dispatch_keyr   rq  is_view2_meta_lib_dont_use_me_use_register_meta_for_mkldnnimpl/_meta_lib_dont_use_me_use_register_meta_for_mkl2_meta_lib_dont_use_me_use_register_meta_for_onednn5_meta_lib_dont_use_me_use_register_meta_for_quantized'_meta_lib_dont_use_me_use_register_meta)activate_meta_tabletypregistryopoop_overloadrJ   s         r7   activate_metar  #  s    9-c2C-+3=#(  9 /446
 k5::#A#ABB+z22 6tK7H6IJ  	EHH00556r:8899 ;
 
 8@@""m $; ;  A     	 [--//BGGXK,,..?DD[U{//11BGGX+"2"2"44EJJ 8<<[Ms 7r9   r   )F)        g      ?rq
  rH   )NNNFr   r2   r   )NNr<  r(  )rA  )rM  T)FF)TT)r  )FTN)TFF)TFrO  )g      ?Nr  r1   )r3   r  r1   F)r3   r  FTN)Fr   FNFr   )NF)r   F)g      ?gUUUUUU?FN)NNNNN)r   NNr2   )NNF)r  FFN)Nr  FFN)r  FNN)Nr  FNNF)r  FN)FN)NNNNNNN)FNNNN)NNNF)NNNNF)Nr   FNN)NNNN)r   TT)NNr   N)d   r   r   )r   )rA   N)r   FF)r  )NTTN(  r  collections.abcr   r   enumr   	functoolsr   typingr   typing_extensionsr   r]   torch._prims_commonrf  rV   r	   r
   r   torch._decompr   r   r   r   
torch._opsr   torch._primsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   torch._prims_common.wrappersr    r!   r"   r#   r$   r  r%   r&   torch.fx.experimentalr'   r  torch.nn.functionalr(   r)   torch.utilsr*   rM   r+   r,   opsr-   libraryLibraryr  r   r  r  r  r8   r>   rQ   r[   rg   rq   linspacelogspacer<  r   taker  r   r   r   r   cummaxcumminr   r   r   r   _fft_c2cr  r  r  _fft_r2cr  randpermgenerator_outr   r   r%  randintr/  r+  low_outr3  randr5  _philox_key_splitr@  _philox_key_fold_inrF  r\  r^  rb  rd  re  _fft_c2rrk  r  rx  r  
unsqueeze_r  _sparse_semi_structured_linearr  _sparse_semi_structured_mmr  _sparse_semi_structured_addmmr  _cslt_sparse_mmr  index_reducer  index_reduce_r  index_selectr  segment_reducer  r  	unary_outr  r   r  r  r  r  r  r  r  _assert_asyncr  msgr  _printr  _make_dep_tokenr  r  _functional_sym_constrain_ranger  r  (_functional_sym_constrain_range_for_sizer  _functional_assert_asyncr  r   r*  r   r9  r@  rH  rR  _linalg_eighrS  r]  _linalg_eigvalslinalg_eigvalsrb  
linalg_eigrg  rl  rq  ry  r  r  r  r  linalg_inv_exr  linalg_ldl_factor_exr  linalg_ldl_solver  	linalg_lur  linalg_lu_factor_exr  linalg_lu_solver  	lu_unpackr  r  	linalg_qrr  r  r  _linalg_svdr  r  rz  r  r*  linalg_solve_triangularr2  r:  rC  _linalg_detrF  r\  rh  r{  reflection_pad1dr  replication_pad1dr  r  reflection_pad1d_backwardr  replication_pad1d_backwardr  r  reflection_pad2dr  replication_pad2dr  _weight_norm_interface_backwardr  reflection_pad2d_backwardr  replication_pad2d_backwardr  r  reflection_pad3dr  replication_pad3dr  reflection_pad3d_backwardreplication_pad3d_backwardr  _pdist_forwardr  _pdist_backwardr  baddbmmr  	bernoullir  
bernoulli_ra   r  r  r  poissonr  _fused_moving_avg_obs_fq_helperr*  mmr6  r  r   rc  rj  miopen_batch_normry  convolutionr  r  _has_mkldnnr  r  _convolution_pointwiser  _linear_pointwiser  has_mklr  r  _mkl_linearr  r  r  qconv2d_pointwiseqconv_pointwiser{  r  binarybinary_tensorr  qlinear_pointwiser  r  linear_dynamic_fp16linear_relu_dynamic_fp16r  r  r  
max_pool2dr  int4mm_packed_weight_cpur  r  quantize_per_tensorr  
avg_pool2dr  r
  avg_pool2d_backwardr  
avg_pool3dr1  avg_pool3d_backwardr=  _adaptive_avg_pool2drC  _adaptive_avg_pool3drG  _adaptive_avg_pool2d_backwardrR  _adaptive_avg_pool3d_backwardrW  rU  adaptive_max_pool2drk  rp  rr  adaptive_max_pool3drz  r|  r}  repeat_interleaver  rz   r  r  r  r   _unsafe_indexr  convolution_backwardr  addbmmr  randint_liker  _fused_adam__fused_adamw_r  _fused_adamr  _int_mmr  _convert_weight_to_int4packr  #_convert_weight_to_int4pack_for_cpur  _weight_int4pack_mmr  _weight_int4pack_mm_for_cpur  r#  r&  r\  _dyn_quant_pack_4bit_weightrh  _dyn_quant_matmul_4bitro  _weight_int8pack_mmrw  _cdist_forwardr  _cdist_backwardr  _embedding_bagr  _embedding_bag_forward_onlyr  r  nansumr  median	nanmedianr  
dim_valuesr  r   r  logical_not_r  repeatr  zero_r  mul_Scalardiv_logical_and_logical_or_logical_xor_r  add_sub_r  r&  subr  rounddecimalsr  r  
__rshift__r  
__lshift__r  zeror   r  r  fillr  relu_r  	_add_relur  rrelu_with_noiser  rrelu_with_noise_functionalr  rrelu_with_noise_r  	index_put_unsafe_index_putr  masked_fill_r  _masked_scaler  masked_scatter_r$  masked_scatterr&  masked_scatter_backwardr(  
index_put_r*  r:  bmmr>  re   	dtype_outr@  rC  rH  r  r  r&  rs  r9  r   max_pool2d_with_indices_backwardr  max_pool2d_with_indicesr  fractional_max_pool2dr  max_pool3d_with_indicesr   max_pool3d_with_indices_backwardr  r  r  r  grid_sampler_2d_backwardr  r  r  r  r  onesr  zerosr  select_scatterr  slice_scatterr  r  r   r  r  gatherr  r+  r3  r5  r@  rC  scatter_addrH  scatter_add_rK  rB  ru  r  r  value_reducerO  scatter_rQ  #_scaled_dot_product_flash_attentionrf  rl  rv  #_scaled_dot_product_cudnn_attentionr  0_scaled_dot_product_fused_attention_overrideabler  9_scaled_dot_product_fused_attention_overrideable_backwardr  ,_scaled_dot_product_flash_attention_backwardr  +_scaled_dot_product_flash_attention_for_cpur  4_scaled_dot_product_flash_attention_for_cpu_backwardr  *_scaled_dot_product_attention_math_for_mpsr  '_scaled_dot_product_efficient_attentionr  0_scaled_dot_product_efficient_attention_backwardr  ,_scaled_dot_product_cudnn_attention_backwardr  _flash_attention_forwardr  +_flash_attention_forward_no_dropout_inplacer  r  _flash_attention_backwardr  _efficient_attention_forwardr  _efficient_attention_backwardr  rE	  
_scaled_mmrH	  r	  _scaled_mm_v2r	  scatter_reducetwotwo_outr	  scatter_reduce_r	  multinomialr	  r	  r	  r	  _upsample_nearest_exact1dr	  _upsample_nearest_exact2dr	  "_upsample_nearest_exact2d_backwardr	  _upsample_nearest_exact3dr   r	  values_stabler	  r
  _thnn_fused_lstm_cellr
  r/
  r2
  r<
  rB
  rF
  rH
  argminrI
  rL
  topkrT
  _segment_reduce_backwardrX
  kthvaluer]
  r   rr
  ro
  rv
  ry
  pixel_shuffler
  r
  	bucketize
Tensor_outr
  
Scalar_outr
  histcr
  _upsample_bilinear2d_aa_upsample_bicubic2d_aa_upsample_lanczos2d_aar
   _upsample_bilinear2d_aa_backward_upsample_lanczos2d_aa_backwardr
  r
  r
  r/  r
  searchsortedr
  r
  embedding_dense_backwardr
  _embedding_bag_backwardr
  _embedding_bag_dense_backwardr
  *_embedding_bag_per_sample_weights_backwardr
  isinr
  	polygammar
  _local_scalar_denser
  r  r  r  rc  _grouped_mmrf  _scaled_grouped_mmri  _foreach_normrq  _softmaxrw  constant_pad_ndr  r  _jagged_to_padded_dense_forwardr  r  _native_multi_head_attentionr  r  special_airy_aispecial_bessel_y0special_bessel_y1special_modified_bessel_i0special_modified_bessel_i1special_modified_bessel_k0special_modified_bessel_k1!special_scaled_modified_bessel_k0!special_scaled_modified_bessel_k1special_chebyshev_polynomial_tspecial_chebyshev_polynomial_uspecial_chebyshev_polynomial_vspecial_chebyshev_polynomial_w&special_shifted_chebyshev_polynomial_t&special_shifted_chebyshev_polynomial_u&special_shifted_chebyshev_polynomial_v&special_shifted_chebyshev_polynomial_wspecial_hermite_polynomial_hspecial_hermite_polynomial_hespecial_laguerre_polynomial_lspecial_legendre_polynomial_pr  r  r  r  lerp_addcmul_addcdiv_torch._refs.nn.functionaltorch._refs.specialr  r3   r9   r7   <module>r     s7    .    '  # + +  " U     < 7 8 ) T]t_yy~~*/--*?*?PV*W ' %a )X"

(* t}}-.
 

==7  /7t 		!!499==12'  3' !!))4+<+<+@+@AB%R %  C%$ t%%&I  'I 	[[$++//4;;+>+>P Xy! " !!))4+<+<+@+@ABI  CI.b  %%t}}'8'89:"<  ;"<J  %%t}}'8'89:@
  ;@
F t}}**+3$ 3 ,3 t}}$$%
 **
 	

 
 
 &
 $$dll&6&678 **	
     9&   $,,"6"678
 **     9& 		!!499==12T $ t PT   3 t%%--.8 /8" t''//0	! 1	!%
P t##++, -
 t$$,,- .
 %%t}}'8'89:*L  ;*LZ tzz!!" #:	( t&&' ( t223) 4)X t../ 0> t112) ) 
) %)) 3)X t##$;< %;<| t  (()	I 	I *	I t!!))*	 	 +	 t  (()' * ' t""**+ W "	 W
 " W " W  W  W  W , WF   $(("4"456  7 txx||    $(("4"456  7 txx||  tzz!!"6 #6 tzz~~( (
 t!!))* + t!!%%& ' t{{""# $ t##++,*
* * 	*
 * * -* t''//0, 1, t33;;< =
 t008896 :6& t<<DDE F
 t,,001 2

  H
 
$ 
  !!))4+<+<+H+HIJ]N+ , K" $$,,d.A.A.E.EFGB  HB  !]N+ , " Q t**+)  ,) t""#J  $J t}})  ) t$$%)  &) t&&../ 0" 	$$,,d.M.M.Q.QR . .d t!!))* + ))1143L3L3P3PQRT8V$  	 % S& %%--t/D/D/H/HIJ'
 '  K'T &&(:(:;<S#st   =6 ((00$2J2J2N2NOPT8V$    	  % Q F $$,,d.B.B.F.FGH4
 4 4  I4n t~~S#s$  $P* &&(:(:;<S#  =4 $$,,d.B.B.G.GHIV[$1' 2 J'$ t''(" )"J.".( t$$%7 	7
 7 !7 7 !7 7 &7t ,,44d6R6R6V6VWX
     Y4 t$$%S#4(+( ) &+(^ t''(
 )
 tzzW  Wt>#;L t$$%=  &= t%%&>  '>(< t--.\S  /S t../\T  0T2Ej t$$%=  &= t%%&>  '> ((00 &&..&&11''//''22	 \& &:<G~ t$$%=  &= t%%&>  '> &&..&&11''//''22	 \(( ((V t""#

  $

 t##$P  %P $$dll&6&678'q ' '  9': &&(:(:;<Id I  =I
 t$$% & t~~ I !I
 $$dll&6&678"  9" t33;;<* =*. tww3  3,B
D DNQ t%%--."$ /"$J t''(# )#L 	889>9N9N&&:6 599##::BBC D, 599##55==>S ?S
 xx:?--:O:O66;
7 
uyy}}00	1	 
2	
 :?9N9N&&:6 599##55==>599##33;;<599##33::;4 < = ?4l 599##55<<=599##55CCD E >< 599##55==>599##55<<= > ?> 599##55<<=599##55CCD$ E >$L 599##77??@599##<<DDE	 F A	 =BMM<Q<QVV=9 599&&112
 3
8 599&&??@@ A@, t''(# )# t&&'M (Mb(<X t''//0E 1EP tUJ   UJp t''(\K(  )K(\ t((001 2" t((001@ 2@ t1199:F ;F, t112\P  3P
	
 	
 t''(UI+  )+\ t001\H  2H$ t''(UI'  )'T t001\(  2(
 t%%,,-* .* $$dll&6&678
  9
 ##++T-@-@-D-DEF

B 

 

  G

 		&&..		0F0F0J0JKL  M" 

!!4#5#5#<#<=>A ?AH ))1123.H 4.Hb ##T[[__56'a 'q '  7'0 !!(()*' +' !!))4+=+=+E+EFG
 
  !
 H
2   (()*! !  !! +!H ~B  B* 0012 3& 889: ; (()*8 +8  0012< 3< >>?@< A<" "@F 0012D 3D4 ++,-; .;( (()*< +< t""**+ & , &F t##$G  %G* t""**+`5 ,`5F t//7785 95
 ##T[[__56=$ =  7= ##T^^%;%;<=) >) !!					 Xy! "	 t  (() * t{{""#' $'& tzz!!" # 								!!  !!

 									**Z  

""DJJ$7$789 :
" &&(>(>?@ A &&(>(>?@ A tyy  !& "& 

!!4::#4#456 7 		  $))"2"234" 5" tzz!!" # t~~$$%   & %%&'"  (" 0012; 3; &&'( ) &&(>(>(F(FGH" I" t  ''( )
 t!!))* + t##$	 %	 t""#6  $6 t++,! -! t&&' (,^   $((,,/05  15  2 234J  5J6;|Y Yx;4|38 38lI2X t445\(  6(V t++334# 5#L t))112Q 3Qh t++,UI_  -_D t445\\  6\~%
 %
Pt  $ t,,445# 6#$ t##$8  %8" t,,-\;'! ( .!, 		!!"#. $. t&&') ()X 		!!499==12  	
     3( 

""DJJNN34  	
     5( t""**+& ,& t!!))*& +&.  /
  t{{""#' $'6
 
%R4 t''(& )&
 t  ! "
 !!	 & & ""	 88@@AB< < C<~ 88BBCD  E4 & 889:( ( ;(V EEFG0 0 H0f tMMN2$ %2 2 O26 99
" "
". 88
 
: AA
!" !"
!"H ??@A- - B-` <<=>)) )) ?))X ==
4- 4-
4-n 99
" "
"0 %%--
J J
JZ @@HHIJ& & K&R --7789' ' :'T &&
, ,
,4 ))
/S /S
/Sd **
47 47
47nAY AYH ''()  *iC iCX ""**+,  -: ##'')<)<)D)DEF&  G&
 t##''( )
   (($*:*:*>*>?@	 	  A	,* 	$$d&D&D&L&LM

 	$$d&D&D&L&LM. ((00//77 : 	$$d&D&D&L&LM

 									*$N t))112 3 t&&'4/ (4/n t&&') ()X t$$,,-$% .$%N


 ##T[[%8%89:4 ;4 t!!))* + tyy  !
Q "
Q t,,-  ." %%t}}';';<=Xy!K " >K  #("9"9 Q t77??@	* A	* t##++,	0 -	0 t!!))* +@ t--556F 7FD %%t~~'@'@AB% u   C %%t~~'@'@AB
 	

 
 
 C
 

|E  E4 $$,,##++##++& --55,,448 t>>FFG H$ '')<)<=>"  ?" uyy~~(() *: uyy~~  !6 "6( t  !-K 	-K
 -K 
-K -K  "-K` t,,- . t++,'
 -'
T t112 3, t>>? @. tyy8 8e 8  8  t~~6  6 t''(I )I tyy"  " t||6  6'TVK VKr t    !& ''()  *6 t!!(() *  t}}    t##$3  %3l t~~8 8  8. t33;;<* * =*, t00889-" :-"` d22 3 d44 5 d44 5 d== > d== > d== > d== > dDD E dDD E tBB C tBB C tBB C tBB C tJJ K tJJ K tJJ K tJJ K t@@ A tAA B tAA B tAA B tyy  $ t||a    t||a   , 	tyy)!$,,/!$,,/
    ENP r9   