o /ѹg@sddlmZddlZddlZddlZddlmZddlmZddl Z ddl Z ddl m Z m Z m Z ddl mZddlmZmZmZmZddlmZdd lmZmZmZzdd lmZWn eyedZYnwz dd lmZmZWn ey}dZdZYnwzdd l m!Z!Wn eydZ!Ynwd Z"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+iZ,dde-e DZ.GdddeZ/GdddeZ0GdddeZ1Gdd d eZ2ej j3e 4d!ej j5e 4d"ej j6e 4d#ej j7e 4d$ej j8eej j9eej j:eiZ;ej j5e jd%e jd%fej j7e jd%e jd%fej j7e jd%e jd%fej j7e jZEdd@dAZFdBdCZG ?   dddQdRZH dddTdUZI  ddd^d_ZJdd`daZKddbdcZLddgdhZMddldmZNGdndodoZOGdpdqdqZPGdrdsdsZQdtduZRdvdwZSdxdyZTdzd{ZUdddZVddZWdddZXdddZYdddZZdddZ[dddZ\dddZ]dddZ^dddZ_dddZ`dddZadddZbdddZcdddZddddZedddZfdddZgdddZhdS)) annotationsN)Enum)Path) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions float8e4m3fn)int4uint4)to_array_extendedz onnx.quantizez0.1.0zai.onnxz com.microsoftQuantizeLinearZ_QuantizeLinear_InputZDequantizeLinearZ_DequantizeLinear_OutputZ _quantizedlcCs(i|]}ttt|trtt||qS) isinstancegetattrrint).0krrh/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/onnxruntime/quantization/quant_utils.py 8s(rc@(eZdZdZdZddZeddZdS)QuantizationModercC|jSNnameselfrrr__str__CzQuantizationMode.__str__cC zt|WStytwr#)r KeyError ValueError)moderrr from_stringF   zQuantizationMode.from_stringN)__name__ __module__ __qualname__Z IntegerOpsZ QLinearOpsr( staticmethodr.rrrrr ? r c@r)QuantizedValueTyperr!cCr"r#r$r&rrrr(Rr)zQuantizedValueType.__str__cCr*r#)r5r+r,)vrrrr.Ur/zQuantizedValueType.from_stringN)r0r1r2ZInputZ Initializerr(r3r.rrrrr5Nr4r5c@sHeZdZdZdZdZdZdZdZdZ dd Z e d d Z e d d ZdS) QuantTyperr!cCr"r#r$r&rrrr(fr)zQuantType.__str__cCr*r#)r7r+r,)trrrr.ir/zQuantType.from_stringcCs|tjkrtjS|tjkrtjS|tjkrtjS|tjkr tj S|tj kr(tj S|tj kr0tj S|tjkr8tjStd|d)NzUnexpected value qtype=.)r7QInt8rINT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r,r&rrr tensor_typeps       zQuantType.tensor_typeN)r0r1r2r?rArGrErCrKrIr(r3r.propertyrMrrrrr7]s r7c@r) QuantFormatrr!cCr"r#r$r&rrrr(r)zQuantFormat.__str__cCr*r#)rOr+r,)formatrrrr.r/zQuantFormat.from_stringN)r0r1r2Z QOperatorZQDQr(r3r.rrrrrOr4rOint8uint8int16uint16dtypeiiiiiiiii@ii@r9zero_point_indexcGsg}t|D]H\}}tt|tjr|t|nt|tjr(||n t d|d|||krN|d}|j tj ksF|j tj krNt d|j qt |dkrYt|S|dS)Nzarg z is not an array: r^zzero_point cannot be r!r) enumeratenumpyZ issubdtypetypenumberappendarrayrndarray TypeErrorrVfloat32float16lentuple)r`argsnew_argsiar6rrr _check_types  rqc Cs|tvs Jd|d|tjjtjjtjjtjjfvr|dkr(td|d|jt j kr2tj }n|jt j kr requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not r>zUnexpected dtype Constant zero_point)valuer)XscaleruYZqurwrx)rwrxF) reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protorrHZFLOAT8E4M3FNUZZ FLOAT8E5M2ZFLOAT8E5M2FNUZNotImplementedErrorrVrbriFLOATrjFLOAT16r,r r r onnxhelper make_tensorr r rqrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundZclip)qTypeZarrrxrulowhighZ onnx_typeZ onnx_modelrefrVqminqmaxZcliplowZcliphighZarr_fp32rrrquantize_nparraysP      rFc Cs|dks|dkrtd|d|t|tjd|jd}t|tjd|jd}|dur;t||tj||jd}|rOtt|t|}| }| }||ks]Jd|d|tj||tj d}tj|tj dtj|tj d}t||} | dksJd| t |jj krtjd |jd} tjd|jd} | | gS|rtjt ||tjd tj d|jd} ntjt ||| |jd} | |j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rUNzqmin=z > qmax=z scale isse?g@)r,rbZminimumrfrVmaximumrrabsZfloat64ZfinfoZtinyrr) rminrmaxrrr{min_real_rangeZabsmaxZdrZdqrxrurrrcompute_scale_zps4   rc sd}|tvr?|tjkr2ddlmddlm}|}fddtdD}tj dd|Dtj d }nt d |d |t|<n |tjkrLddlm}|}|durXt d |d t t|}tj d|d }tj |||jd }||gS)arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nrfloat8e4m3_to_float32rcsg|]}|qSrr)rrorrr Jsz+compute_scale_zp_float8..cSs$g|]}t|st|s|qSr)rbisnanisinf)rfrrrrLs$rUzQuantization to element_type=z not implemented.zUnexpected element_type r>)FLOAT8_DISTRIBUTIONSrrHZonnx.numpy_helperr#onnx.reference.custom_element_typesrrangerbrfrir,rhstdrV) Z element_typerZzp_dtyperZ all_valuesvaluesZstd_f8zerorxrrrcompute_scale_zp_float89s*       rdata numpy.ndarray quant_typeonnx.TensorProto.DataTyper{boolrzr float | None rmin_override rmax_overridereturn#tuple[numpy.ndarray, numpy.ndarray]cCsPt|tjstdt|d|dur|}n t|r|nd}|dur(|}n t|r0|nd}tj||j d}tj||j d}tjd|j d} |t j krh|rUt dt |} t|| \} } t| | dd S|t jt jt jt jt jt jfvrt|||d \} } t|rt||| | ||\} } ntjd| j d} t| | dd Std |d) a Returns the zero_point and scale for the given data. :param data: The data for which to compute quantization parameters. :param quant_type: The quantization data type. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: zero point and scale z%Weight must be given as an array not r>NgrUrz1Unsupported option reduce_range=True for float 8.rr_r{z Unexpected value for quant_type=)rrbrgrhrcrkrrrfrVrrH RuntimeErrorrrrqr@rBrFrDrLrJrrr,)rrr{rzrrrrrrxrrurrrrrcompute_data_quant_params^s>   r2tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]c Cst|||||||\}}|tjkrIt||||} t| tjd@dkrDt |} t d| d| d| d| d ||| fS|tj tjtjtjtjtjfvret||||} ||| fStd|d) al :param data: data to quantize :param qType: data type to quantize to. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: minimum, maximum, zero point, scale, and quantized weights To pack weights, we compute a linear transformation - when data `type == uint8` mode, from `[rmin, rmax]` -> :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point rXz+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=r>N)rrrHranyrrbrRravelrrrrr@rBrFrDrLrJr,) rrr{rzrrrrurxquantized_dataZnp_datarrr quantize_datas@    rweightonnx.TensorProtorurxaxis int | Nonequant_weight_name str | Nonec Cst|}d}|durt||||}n?|j|}t|j} d| |<g} t|D]$} || |} || } || }t|| | |}| t | | q(t | |}|rW|n|j t }|tjjkrt}||_|j|j||_ ||_tdurt|}|j|jks||krtd|jd|ddd|ddd|jdt|dd d |S|tjjtjjfvr|jtjtjfvrtd |d t t!|}tj"j#|||j|d d}|Stj"$|}tj ||d |j}tj%&||}|S)aG Returns a quantized version of the given ONNX initializer. :param weight: The ONNX initializer to quantize. :param quant_type: The final quantized data type. :param zero_point: The zero-point value to use for quantization. :param scale: The scale value to use for quantization. :param axis: The quantization axis if quantizing per-channel. Defaults to None. :param quant_weight_name: The name of the quantized initializer. If not specified, the quantized name is generated. :return: The quantized ONNX initializer. Nr!zThe initializer of shape z! could not be created, expecting z, got z and shape=z raw=r>zQuantized weights for z. must be 8-bit before packing as 4-bit values.T)rawrU)'tensor_proto_to_arrayrrshapelistrZtakererbrZreshapeZ concatenater%TENSOR_NAME_QUANT_SUFFIXrrrH data_typedimsextendflattencopytobytesraw_datarrstrrLrJrVrQrRbytespack_bytes_to_4bitrrZtensor_dtype_to_np_dtype numpy_helperZ from_array)rrrurxrrZ weight_dataZ q_weight_dataZ channel_countZ channel_dimsZquantized_channel_data_listroZ channel_dataZ channel_scaleZchannel_zero_pointZquantized_channel_dataZ q_weight_nameZq_weight_initializercheckZ packed_dataZquant_np_dtyperrrquantize_onnx_initializersb         rc Cs|tjjkr tdd}|rt|}n|r|tvrt|}nt|}|s.td|d|\}}|dks:|dkrQtd|d|d|j d |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrrrsrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) r~rrHrONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGEONNX_INT_TYPE_RANGEr,rV)rrzr{Zqrangerrrrrr(s8     rcCst|||d\}}||S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. rN)r)rrzr{rrrrrget_qrange_for_qTypeHsrrranktuple[bool, int]cCs,|dkr||n|}|dko||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rNr)rrZ axis_normZis_validrrrnormalize_axisRsrsrc_8bitr bytearraycCst|}|dkr tS|dd}t|}d}d}||dkr?||dd@d>||d@B||<|d7}|d7}||dks||krK||d@||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rr!r8rYr:N)rkr)rZ num_elemsZdst_sizedstZsrc_iZdst_irrrr^s  $ rc@s eZdZdZggdfddZdS)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc C:||_||_||_||_||_||_||_||_| |_dSr#) r% initializerrminsrmaxs zero_pointsscalesrrr) r'r%rrrrrrrrrrr__init__s  zQuantizedInitializer.__init__r0r1r2__doc__rrrrrr|s  rc@s"eZdZdZ    dddZdS)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc Crr#) original_nameZq_name scale_nameZzp_nameZ value_typer node_type node_qtype scale_type) r'r%Znew_quantized_namerZzero_point_nameZquantized_value_typerrrrrrrrs  zQuantizedValue.__init__)NNNNrrrrrrs rc@seZdZdZddZdS)BiasToQuantizez+ Represents a bias to be quantized cCs||_||_||_dSr#) bias_name input_name weight_name)r'rrrrrrrs zBiasToQuantize.__init__Nrrrrrrs rcCs|jdkrtd|jd|jdkr|j}n^|jdkr |j}nU|jdkr)|j}nL|jdkr2|j}nC|jdkr;|j}n:|jd krD|j}n1|jd krM|j }n(|jd krV|j }n|jd kr_|j }n|jd krh|j }n td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.r!r8r9r:r;r<rZ rz has unsupported type r>N) rcr,r%rrosr=gfloatsZintsstringsZtensorsZgraphs) attributervrrrattribute_to_kwargs0            rcs*fdd|D}t|dkr|dSdS)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. csg|] }|jkr|qSrr$)ritem item_namerrrsz find_by_name..rN)rk)rZ item_listitemsrrr find_by_namesrcCs*d}tt|D] }|||kr|}q|S)zC Helper function to return index of an item in a node list r^N)rrk)Z elem_nameZ elem_listZelem_idxrorrrget_elem_indexs  rcCstjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. ZMulN)rrr )inputsoutputr%rrr get_mul_nodesrfilenamer identifierrcCs|j|j||jS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. N)parentjoinpathstemsuffix)rrrrrgenerate_identified_filenamesr cCsddl}ddlm}ddl}|j|jdtdt|tdt||j||dd|d| d| d | dS) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueZCountszTensor value V.S. Counts) sysZmatplotlib.pyplotZpyplotrbZset_printoptionsmaxsizeprintZstairsZxlabelZylabeltitleshow)histZ hist_edgesrZpltrbrrr apply_plots     rr>cs<ddlddl}ddlddlmmm}ddlmmm}ddl m m m t d|Gfdddj}j||d}ttj|dd  }||Wdn1sfwYd}|d } g} t|D]I} || } | } t| d |t| d |g}tt|}| | }| |}|!| |"| ||#| ||$| }| %|q}|&| t'| | D]}| (|q| )}|*| |+| ||,| }| -|| .}ttj|d d }||Wdn 1swYtj/dddvrF|j0|d}|1}t2|D]}|3|}t |4t |5q/ttj|dd C}t|D]3} || } | } t| d |t| d |g}| dtt|}|||dqWWddS1swYdS)z> Helper function to write calibration table to files. rN)CalibrationMethod TensorData TensorsDatazcalibration cache: cs eZdZfddZdS)z*write_calibration_table..MyEncodercsbt|fr |St|jr|t|jddSt|r*|jjt|dSj ||S)Nz numpy.array)rrVCLS)rrv) rto_dictrgtolistrrV __class__r0 JSONEncoderdefault)r'objrrrjsonnprrr2s  z2write_calibration_table..MyEncoder.defaultN)r0r1r2rrr rr MyEncoder1sr#)clszcalibration.jsonwiZhighestZlowestzcalibration.flatbufferswbZQUANTIZATION_DEBUG)r!1zcalibration.cache  )6r! flatbuffersrbZ5onnxruntime.quantization.CalTableFlatBuffers.KeyValueZ quantizationZCalTableFlatBuffersKeyValueZ5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTableZ"onnxruntime.quantization.calibraterrrlogginginfordumpsopenospathjoinwriterfZBuildersortedkeysrfloatrrrrZ CreateStringZ KeyValueStartZKeyValueAddKeyZKeyValueAddValueZ KeyValueEndreZTrtTableStartDictVectorrkZPrependUOffsetTRelativeZ EndVectorZ TrtTableStartZTrtTableAddDictZ TrtTableEndZFinishZOutputenvironZGetRootAsTrtTableZ DictLengthrDictKeyValue)Zcalibration_cachedirr*r+r,r#Z json_datafilerZbuilderZkey_value_listkeyrZd_valuesrrvZflat_keyZ flat_value key_valueZ main_dictZ cal_tablebufZdict_lenrorr rwrite_calibration_table!sx                     $rA-C6?cCs|dktj}|dktj}|}|j|}|sdS|t|t|}|dks4Jd|||f|tj}|||| |7}|dkdksOJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrz"n_zeros=%d, n_nonzeros=%d, eps1=%f)rrbrisumsizer7)pZepsZis_zerosZ is_nonzerosZn_zerosZ n_nonzerosZeps1rrrrsmooth_distribution}s    rF model_pathcCs4tj|dd}|jjD] }t|rdSq dS)NF)Zload_external_dataT)rloadas_posixgraphrrZuses_external_data)rGmodelZ intializerrrrmodel_has_external_datas   rLopt_model_pathcCsFt}||_tj|_i}dg|d<t||fddgi|}dS)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ZConstantSharingZdisabled_optimizers providersZCPUExecutionProviderN)rrIZoptimized_model_filepathrZORT_ENABLE_BASICZgraph_optimization_levelr)rGrMZ sess_optionkwargs_rrroptimize_models    rQrKrcCs>ddi}|jr|jD] }||j|jiq tj||dS)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdater>rvrrZset_model_props)rKrUproprrradd_pre_process_metadatas  rXcC0|jr|jD]}|jdkr|jdkrdSqdS)zCCheck the model whether it went through quantization pre-processingrRrSTFNrUr>rv)rKrWrrrmodel_has_pre_process_metadatas  r[cCs>ddi}|jr|jD] }||j|jiq tj||dS)N onnx.inferrSrT)rKrUrErrradd_infer_metadatas  r]cCrY)Nr\rSTFrZ)rKrErrrmodel_has_infer_metadatas  r^cCsBt|d}tjt|t|t|}t|||S)Nz -inferred) r rZshape_inferenceZinfer_shapes_pathrrHrIr]unlink)rGZinferred_model_pathrKrrrload_model_with_shape_infers r`cCsZtjdd}t|d}tj||ddt|WdS1s&wYdS)Nz ort.quant.)prefixz model.onnxT)Zsave_as_external_data)tempfileTemporaryDirectoryrr rZ save_modelrIr`)rKZ quant_tmp_dirrGrrr&save_and_reload_model_with_shape_infers $rdrrcCs>|jtjjtjjfvrtj|Std|j dt |j)Nz&Only float type is supported. Weights z is ) rr~rrrrrZto_arrayr,r% type_to_name)rrrrrs  r tensor_namecC|dS)NZ_QuantizeLinearrrfrrradd_quant_suffixricC|tSr#)QUANT_INPUT_SUFFIXrhrrradd_quant_input_suffixrjrmcCrg)NZ_QuantizeLinear_Outputrrhrrradd_quant_output_suffixrjrncCrg)NZ_DequantizeLinearrrhrrradd_dequant_suffixrjrocCrg)NZ_DequantizeLinear_Inputrrhrrradd_dequant_input_suffixrjrpcCrkr#)DEQUANT_OUTPUT_SUFFIXrhrrradd_dequant_output_suffixrjrr)NN)FN)FNNN)rrrrr{rrzrrrrrrrrr)rr)rrrrrurrxrrrrrrr)FF)rrrrrr)rrrr)rrrrrr)r>)rB)rGr)rGrrMr)rKr)rKrrr)rGrrr)rKrrr)rrrr)rfrrr)rr)i __future__rr-r1rbenumrpathlibrrbrrrrrr~Z onnx.helperr r r r Zonnx.referencer Z onnxruntimerrrrr ImportErrorrrZonnx.reference.op_runrZ __producer__ __version__Z onnx_domainZ ms_domainZ QUANT_OP_NAMErlZDEQUANT_OP_NAMErqrZMODEL_SIZE_THRESHOLDrr<rer r5r7rOr@rVrBrFrDrHrLrJr}rfrRrQrTrSrrrrqrrrrrrrrrrrrrrrrrr rrArFrLrQrXr[r]r^r`rdrrirmrnrorprrrrrrs        &$ $$$  $ $$$$$$$    4?) ? B O  %    \