o *ѹg@@sUddlZddlmZddlmZmZmZddlmZm Z m Z m Z m Z m Z ddlmZddlmZddlmZddlmZddlmZdd lmZd d gZejjZd d ZiZe e e fed<ddZ d?dd Z!e!ej"ddde#fddZ$e!ej%d@de#fddZ&e!ej'd@de#fddZ(e!ej)d@de#fddZ* d?dee#dee#dee#d e+de#f d!d"Z,e!ej-ej.gddde#fd#d$Z/d%d&Z0e!ej1de#fd'd(Z2d)d*Z3e!ej4ej5gddde#fd+d,Z6d-d.Z7e!ej8ej9gddde#fd/d0Z:ej"e$ej%e&ej'e(ej)e*ej-e/ej.e/ej1e2ej4e6ej5e6ej8e:ej9e:i Zd1d2Z;gd3Zd8d9Z?d:d;Z@Gdd>e ZBdS)AN)tree_map tree_flattentree_unflatten)ListAnyDictOptionalUnion NamedTuple) defaultdict)TorchDispatchMode)RemovableHandle)register_decompositionprodwrapsFlopCounterModeregister_flop_formulacCst|tjr |jS|SN) isinstancetorchTensorshape)ir\/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/torch/utils/flop_counter.py get_shapes r flop_registrycstddfdd }|S)N)outcs(tt|||f\}}}|d|i|S)N out_shape)rr)rargskwargsr frrnfszshape_wrapper..nfrr$r%rr#r shape_wrappersr'Fcfdd}|S)Ncs"st|}ttdd||S)NT)registryunsafe)r'rr)Z flop_formulaget_rawtargetsrr register_fun!sz+register_flop_formula..register_funr)r-r,r.rr+rr s)r returnc Os,|\}}|\}}||ksJ||d|S)zCount flops for matmul.Nr) a_shapeb_shaper r!r"mkk2nrrrmm_flop)s r7cK t||S)zCount flops for addmm.N)r7Z self_shaper1r2r r"rrr addmm_flop4s r:c KsD|\}}}|\}}} ||ksJ||ksJ||| d|} | S)z"Count flops for the bmm operation.r0Nr) r1r2r r"br3r4b2r5r6floprrrbmm_flop9s    r>cKr8)z&Count flops for the baddbmm operation.Nr>r9rrr baddbmm_flopFs r@x_shapew_shaper transposedc CsJ|d}|r|n|dd}|^}}}|t||t|d|} | S)aCount flops for convolution. Note only multiplication is counted. Computation for bias are ignored. Flops for a transposed convolution are calculated as flops = (x_shape[2:] * prod(w_shape) * batch_size). Args: x_shape (list(int)): The input shape before convolution. w_shape (list(int)): The filter shape. out_shape (list(int)): The output shape after convolution. transposed (bool): is the convolution transposed Returns: int: the number of flops rr0Nr) rArBr rCZ batch_sizeZ conv_shapeZc_outZc_indimsr=rrrconv_flop_countNs   rEc Ost||||dS)zCount flops for convolution.)rCN)rE) rArB_bias_stride_padding _dilationrCr r!r"rrr conv_flopksrJcCs |d|dgt|ddS)Nrr0)list)rrrrtranspose_shapeps rMc CsZd} | drt| d} | t||| | 7} | dr+t| d}| tt||||7} | SNrrK)rrErM)grad_out_shaperArBrFrGrHrIrCZ_output_paddingZ_groupsZ output_maskr flop_countZgrad_input_shapeZgrad_weight_shaperrrconv_backward_flopss  rQcCs|\}}}}|\}}} } |\} } } }||kr| kr8nJ||kr)| kr8nJ|| kr8| | kr8|| ks:Jd}|t||||f|||| f7}|t|||| f||| |f7}|S)z^ Count flops for self-attention. NB: We can assume that value_shape == key_shape rNr?) query_shape key_shape value_shaper;hs_qd_q_b2_h2s_k_d2_b3_h3_s3d_v total_flopsrrrsdpa_flop_counts   P""racOs t|||S)zCount flops for self-attention.N)ra)rRrSrTr r!r"rrr sdpa_flops rbcCsVd}|\}}}}|\} } } } |\} }}}|\}}}}|| kr)| kr)|krBnJ|| kr;|kr;|krBnJ|| ksDJ||krP| |krP||ksRJd}|t||||f|||| f7}|t||||f|||| f7}|t||| |f||||f7}|t|||| f||| |f7}|t||||f|||| f7}|S)Nrr?)rOrRrSrTr`r;rUrVrWrXrYrZr[r\r]r^r_Z_b4Z_h4Z_s4Z_d4rrrsdpa_backward_flop_counts    T"""""rccOst||||S)z(Count flops for self-attention backward.N)rc)rOrRrSrTr r!r"rrrsdpa_backward_flopsrdcCst|ts|fS|Sr)rtuplexrrrnormalize_tuples rh)KMBTcCs0tdtttdtt|dd}t|S)NrrK)maxminlensuffixesstr)numberindexrrrget_suffix_strs(rvcCs&t|}|d|d}|t|S)Niz.3f)rrru)rtsuffixruvaluerrrconvert_num_with_suffixs  rycCs|dkrdS||dS)Nr0%z.2%r)numZdenomrrrconvert_to_percent_strs r|cstfdd}|S)Ncst|\}}|}t||Sr)rr)r!Z flat_argsspecrr#rrr%s  z)_pytreeify_preserve_structure..nfrr&rr#r_pytreeify_preserve_structuresr~c seZdZdZ    d%deeejje ejjfde de dee e e ffd d Zd d Zd dZddZddZddZddZde fddZde ee e e fffddZd&ddZfddZfd d!Zd'd#d$ZZS)(ra ``FlopCounterMode`` is a context manager that counts the number of flops within its context. It does this using a ``TorchDispatchMode``. It also supports hierarchical output by passing a module (or list of modules) to FlopCounterMode on construction. If you do not need hierarchical output, you do not need to use it with a module. Example usage .. code-block:: python mod = ... flop_counter = FlopCounterMode(mod) with flop_counter: mod.sum().backward() Nr0Tmodsdepthdisplaycustom_mappingcCsntdd|_||_dg|_||_|duri}t|tjjr!|g}||_ i|_ it dd| D|_ dS)NcSsttSr)r intrrrrsz*FlopCounterMode.__init__..GlobalcSs*i|]\}}|t|ddr|nt|qS)Z_get_rawF)getattrr').0r4vrrr !s*z,FlopCounterMode.__init__..) r flop_countsrparentsrrrnnModuler_module_to_forward_hook_handlesritems)selfrrrrrrr__init__s zFlopCounterMode.__init__cCs|jdurdS|jD]:}t|j}t|D]*\}}|dkr$|}nd||g}|||}| | |}t |||j |<qq dS)Nri.) rtype__name__dictZ named_modulesrjoinZregister_forward_pre_hook _enter_moduleZregister_forward_hook _exit_module_ForwardHookHandlesr)rmodprefixnamemoduleforward_pre_hook_handleforward_hook_handlerrr_register_forward_hooks$s    z'FlopCounterMode._register_forward_hookscCs6|jD]}|d|dq|jdSrN)rvaluesremoveclear)rZforward_hook_handlesrrr_deregister_forward_hooks5s z)FlopCounterMode._deregister_forward_hookscr()Ncst|}|Sr)r~_create_pre_module)rinputsrrrrrr$<z(FlopCounterMode._enter_module..frrrr$rrrr;szFlopCounterMode._enter_modulecr()Ncst|}|Sr)r~_create_post_module)rroutputsrrrr$Crz'FlopCounterMode._exit_module..frrrrrrBszFlopCounterMode._exit_modulec Gfdddtjj}|jS)Nc0eZdZefddZefddZdS)z6FlopCounterMode._create_post_module..PushStatecs.jdks Jjtdd|}|S)NcSt|tjr |S|SrrrrclonerfrrrrNzPFlopCounterMode._create_post_module..PushState.forward..)rpoprctxr!rrrforwardJs z>FlopCounterMode._create_post_module..PushState.forwardcsj|Sr)rappendrZ grad_outsrrrbackwardQs z?FlopCounterMode._create_post_module..PushState.backwardNr __module__ __qualname__ staticmethodrrrrrr PushStateIs rrZautogradZFunctionapply)rrrrrrrH z#FlopCounterMode._create_post_modulecr)Ncr)z4FlopCounterMode._create_pre_module..PopStatecsjtdd|}|S)NcSrrrrfrrrr]rzNFlopCounterMode._create_pre_module..PopState.forward..)rrrrrrrrZs z.PopState.forwardcs jdks Jj|S)Nr)rrrrrrr`s z=FlopCounterMode._create_pre_module..PopState.backwardNrrrrrPopStateYs rr)rrrrrrrXrz"FlopCounterMode._create_pre_moduler/cCst|jdS)Nr)sumrrrrrrget_total_flopshszFlopCounterMode.get_total_flopscCs t|jS)aReturn the flop counts as a dictionary of dictionaries. The outer dictionary is keyed by module name, and the inner dictionary is keyed by operation name. Returns: Dict[str, Dict[Any, int]]: The flop counts as a dictionary. N)rrrrrrget_flop_countsks zFlopCounterMode.get_flop_countsc s|durj}|dur d}ddl}d|_gd}g}tdfdd}jD]$}|dkr9q2|d d }||krEq2|||d }|D]} || qNq2djvrzszt |D]\} } d || d|| d<qb|dd|}t |dkrgd g}|j||d dS)Ni?BrT)rZFLOPz% TotalFcstj|}|kOd|}g}|||t|t|gj|D]\}}||dt|t|t|gq,|S)N z - )rrrrryr|rrs)mod_namerr`paddingrr4rZ global_flopsZ global_suffixZis_global_subsumedrrr process_mods  z.FlopCounterMode.get_table..process_modrrrKr)r0rz)leftrightr)headersZcolalign) rtabulateZPRESERVE_WHITESPACErrvrkeyscountr enumeraterq) rrrheaderrrrZ mod_depthZ cur_valuesrxidxrrr get_tablews:   zFlopCounterMode.get_tablecs |j|t|Sr)rrrsuper __enter__r __class__rrrs  zFlopCounterMode.__enter__cs.|jr t||j|tj|dSr)rprintrrrr__exit__)rr!rrrrszFlopCounterMode.__exit__rc Csp|r|ni}||i|}|j}||jvr6|j|}||i|d|i}|jD] } |j| ||7<q(|S)Nr)Z_overloadpacketrrr) rfunctypesr!r"rZ func_packetZflop_count_funcrPparrrr__torch_dispatch__s    z"FlopCounterMode.__torch_dispatch__)Nr0TNr)rN)rrr__doc__rr rrrrrboolrrrrrrrrrrrsrrrrr __classcell__rrrrrs6  = c@seZdZUeed<eed<dS)rrrN)rrrr __annotations__rrrrrs  r)Fr)CrZtorch.nnrZtorch.utils._pytreerrrtypingrrrrr r collectionsr Ztorch.utils._python_dispatchr Ztorch.utils.hooksr Z torch._decomprmathr functoolsr__all__opsZatenrrrr'rmmrr7Zaddmmr:Zbmmr>Zbaddbmmr@rrEZ convolutionZ _convolutionrJrMZconvolution_backwardrQraZ'_scaled_dot_product_efficient_attentionZ#_scaled_dot_product_flash_attentionrbrcZ0_scaled_dot_product_efficient_attention_backwardZ,_scaled_dot_product_flash_attention_backwardrdrhrrrvryr|r~rrrrrrs                T