o åiÿ<ã@säddlZddlmZmZmZddlmZeddd„iƒedejd ejd ejdejdejf dd „ƒƒZddd„Z dd„Z edejd ejd ejdejdejf dd„ƒZddd„Zdd„Z ddd„ZGdd„dejjƒZGdd„dƒZdS) éNé)ÚcdivÚ heuristicsÚjit)ÚlanguageÚEVEN_KcCs|d|ddkS)NÚKÚTILE_Kr©)Únargsr r úl/var/www/html/eduruby.in/lip-sync/lip-sync-env/lib/python3.10/site-packages/triton/ops/blocksparse/matmul.pyÚór ÚTILE_MÚTILE_Nr ÚBLOCKc*Cs<t d¡|}||d7}t d¡}t |d¡}t |d¡}||t d|¡|}t d|¡}||||||dd…df||ddd…f|}t |d¡}||t d|¡|}t d|¡} ||||||ddd…f| | dd…df| }!tj||ftjd}"t|d|ƒD]F}#|r§t |¡}$t |!¡}%n tj||ddd…f|#kdd}$tj|!| dd…df|#kdd}%|"tj|$|%tjd7}"|||7}|!|| 7}!q˜|" |j j ¡}&t d|¡|}'t d|¡|}(||||||'dd…df| |(ddd…f|})tj|)|&d d dS)Nrréé©Údtypeg)ÚmaskÚother©Z out_dtypeT©r)ÚtlÚ program_idÚloadÚarangeÚzerosÚfloat32ÚrangeÚdotÚtorÚ element_tyÚstore)*ÚAÚBÚCZ stride_zaÚ stride_haZ stride_maÚ stride_akÚ stride_zbÚ stride_hbÚ stride_bkZ stride_nbÚ stride_zcÚ stride_hcZ stride_mcZ stride_ncrZgrid_offsetÚlutrrr rrÚblock_idZoff_zÚoff_hZstart_amÚoffs_amÚoffs_akZa_ptrsZstart_bnÚoffs_bnÚoffs_bkZb_ptrsÚaccÚkÚaÚbÚcÚoffs_cmÚoffs_cnÚpcr r rÚ_sdd_kernelsf ÿþýüÿþýü ÿþýür>c CsÆ| d¡dkr| d¡dkr| ¡}| d¡dkr$| d¡dkr$| ¡}|r2||}}||}}|r6dnd} |r)r8r9Útrans_aÚtrans_bÚtrans_cÚspdimsÚblockr/ÚwidthsÚoutZa_dimZb_dimZKaZKbr:Úgridr r rÚ sdd_matmulTs6 *"00 ù rVcCs&|jdd |¡ ¡}| ¡}|dfS)NF©Úas_tuple)Únonzeror"ÚintrI)ÚlayoutrRrCr/r r rÚsdd_lutvsr\ÚGROUP_SIZE_Mc4Csüt d¡}t d¡}t d¡}t d¡}t |||||¡\}}t d¡}||d}t |d¡}t |d¡}t |d¡}t |d¡} ||}!t |!d¡}"t |"d¡}"t d|¡}#t d|¡}$||||"||#dd…df||$ddd…f|}%||t d|¡}&t t |&||¡|¡}&t |!¡}'t |'d¡}'|'t d|¡}(|||| ||&ddd…f| |(dd…df| })tj||ftj d}*|!d7}!t |!d¡}+t |+d¡}+t |!¡},t |,d¡},t |d|ƒD]=}-t |%¡}.t |)¡}/|*tj|.|/tj d7}*|%|+7}%|)|,| 7})|!d7}!t |!d¡}+t |+d¡}+t |!¡},t |,d¡},qõ|* |j j¡}0||t d|¡}1||t d|¡}2|| ||||1dd…df| |2ddd…f|}3tj|3|0|2ddd…f|kd dS) NrrrrErérrr)rrZnum_programsZ swizzle2drZmultiple_ofrZmax_contiguousrrr r!r"rr#r$)4r%r&r'Z stride_azr(Z stride_amr)r*r+r,Z stride_bnr-r.Z stride_cmZ stride_cnZDS0ZDS1r/rrr r]rZpid_mZpid_nZ num_pid_mZ num_pid_nZpidzÚheaderÚoffsetrÚcolumnr1Zpincr0r2r3Úpar4Zstart_bkr5Zpbr6Zinc_aZinc_br7r8r9r:r;r<r=r r rÚ_dsd_kernel‚s‚ ÿþý ÿþý ÿþýü$rcc s¾| d¡dkr| d¡dkr| ¡}| d¡dkr$| d¡dkr$| ¡}|||r*dnd} | d¡‰| d¡}| |r>dnd¡‰|j}ˆ} |}|rLˆn| }|rR| nˆ}| durftj| |||f||jd}n | j| |||fksqJ‚| }d}‡‡‡fdd„}t||||| d¡| d¡| |r‘dnd¡| |r™dnd¡| d¡| d¡| |r©dnd¡| |r±dnd¡| d¡| d¡| |rÁdnd¡| |rÉdnd¡ˆ| |f||t |d ƒ|d d d dœŽ|S)NrrrrrBé€cstˆ|dƒˆˆgS)Nr)r)Úmeta©ÚBS0ZBS3Úwidthr rr ærzdsd_matmul..rDrE)rrr rrFrGr]) rHrIÚsizerrLrMrCrJrcÚmin)r8r9rNrOrPrQrRr/rhrTZAS1ÚBS1rZCS0ZCS1ZCS2ZCS3r:rrUr rfrÚ dsd_matmulÎs< 000ûù rlcCsžt ||rdnd¡}t |¡jdd\}}| ¡}||}|r&|jdd} n | dd¡jdd} | d¡} t |¡}tj|dd…dd |dd…<t || dt |¡¡}| dd…df|}| ¡} | dd…|dd…8<||}| dd¡ d|¡} || dd…dd…f<| dd…df|d|8<|||dk| ||dkdf<| d¡} |rºtj | |jd }nLtjgtj|jd}d}t| d¡ƒD]8}||dd…dd…f ¡ ¡}| ¡}dtj ||jd ||dk<t |||j|jdkdf¡}||7}qÍ|||}|dd…|dd…||8<| dd¡ d|¡}|rF||dd…dd…f<|dd…df|d|8<n|||dd…dd…f<|dd…df|d||8<|||dk|||dkdf<| d¡}| d¡}|d|d|}||}tj||||fdd d¡ ¡}tj| |fdd d¡ ¡}tjd |j|jd}t ||f¡}t ||f¡}| tj¡ |¡}||fS)a Generates the look-up table for incrementing pointers in the DSD/DDS matmul. Example (BLOCK=32, STEP=16) [[1, 0, 0, 1, 0], [0, 1, 1, 0, 1], [1, 0, 1, 0, 0]] Then the offsets for A are [0 , 16, 32, 48] <- row 0 \----/ \----/ col=0 col=3 [64, 80, 96, 112, 128, 144] <- row 1 \----/ \----/ \------/ col=1 col=2 col=3 [160, 176, 192, 208] which leads to increments table [0, 16, 16, 16, || 64, 16, 16, 16, 16, 16, || 160, 16, 16, 16] Because B is dense, the offsets are [0, 16, 96, 112] <- row 0 [32, 48, 64, 80] <- row 1 [0, 16, 64, 80] <- row 2 rrTrWFrNr@)Údim)rCrBrEé)rCr)rLÚsumZ ones_likerYÚflattenZ transposeriZ zeros_likeZcumsumrjÚcloneÚviewÚrepeatrrCZtensorZint64r ÚlongÚcatÚTÚstackrIrrÚtypeZint32r")r[rRÚstepZtransrCÚsizesZhead_idZcol_idÚsegmentsZnnzZ num_blocksÚoffsetsZB_idxZB_incsÚdivZA_idxÚcurrent_offsetÚzZlayoutwZmsumZA_incsrhr_ZincsÚpadr/r r rÚdsd_lutôsd " $"$ rc Cs"t|||||||||| d S©N)rT)rl) r8r9rNrOrPrQrRr/rhrTr r rÚ dds_matmulZs"rƒc@s0eZdZeeedœZedd„ƒZedd„ƒZ dS)Ú_matmul©ÚsddÚdsdÚddscCsxtj||||||||| | |d }| ||¡||_||_| |_||_||_||_||_ ||_ ||_||_|du|_ |Sr‚)r„ÚfnZsave_for_backwardÚda_lutÚda_widthÚdb_lutÚdb_widthÚmoderQrRrNrOrPÚhas_out)Úctxr8r9rNrOrPrŽrQrRÚc_lutÚc_widthrŠr‹rŒrrTr:r r rÚforwardfs" z_matmul.forwardc Csè|j\}}d\}}|j}|jdr4|d|d|d}tj||||j|j|j|j|j |j |jƒ }|jdr\|d|d|d}tj||||j|j|j|j|j |j|j ƒ }|jra|nd} ||dddddddddddd| fS)N)NNrrr)Z saved_tensorsrŽZneeds_input_gradr„r‰rPrOrNrQrRrŠr‹rŒrr) rZdcr8r9ÚdaÚdbrŽZmode_daZmode_dbZdoutr r rÚbackward{s$ "ÿ "ÿ þz_matmul.backwardN) Ú__name__Ú __module__Ú__qualname__rVrlrƒr‰Ústaticmethodr“r–r r r rr„bs r„c@s eZdZddd„Zddd„ZdS) ÚmatmulFc CsD|dvrtdƒ‚||_||_||_||_||_||_|j|_t |dƒ}|jdkrJt |||ƒ\|_|_t |||d|ƒ\|_|_t |||d|ƒ\|_|_|jdkrtt ||||j|ƒ\|_|_t |||ƒ\|_|_t ||||j|ƒ\|_|_|jdkr t ||||j|ƒ\|_|_t ||||j|ƒ\|_|_t |||ƒ\|_|_dSdS) Nr…z"Supported modes are: sdd, dsd, ddsrDr†TFr‡rˆ)ÚNotImplementedErrorrRrŽrNrOrPr[rJrQrjr\r‘r’rrŠr‹rŒr) Úselfr[rRrŽrCrNrOrPryr r rÚ__init__•s. ýzmatmul.__init__NcCsBt |||j|j|j|j|j|j|j|j |j |j|j|j |¡}|S©N)r„ÚapplyrNrOrPrŽrQrRr‘r’rŠr‹rŒr)rr8r9rTr:r r rÚ__call__sûzmatmul.__call__)FFFrŸ)r—r˜r™ržr¡r r r rr›“s r›rŸ)rLÚrrrrrZ constexprr>rVr\rcrlrrƒZautogradÚFunctionr„r›r r r rÚsJÿúúúùù @"úúúùù K& f1