o /ѹg @sddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZddlZddlZddlmZmZmZmZddlZddlmZmZmZdejd ejd ejfd d Z  d0dejd ejd e eded ejf ddZGdddZ GdddZ!GdddeZ"Gdddej#dZ$GdddZ%Gddde%Z&Gddde%Z'Gd d!d!e'Z(Gd"d#d#e'Z)Gd$d%d%e'Z*Gd&d'd'ej#dZ+Gd(d)d)e+Z,dd*e"j-d+ifd,ee.efd-e e e.fd.d/Z/dS)1N)Enum)Path)DictOptionalSequenceTupleUnion) ModelProto TensorProtohelper numpy_helper) apply_plotload_model_with_shape_infersmooth_distributionpkqkreturncCs|tj|j|jd}|ddt|dd|dd|dd<|dk|dk@}d||<|dk|dk@}tj||<|S)z See https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.rel_entr.html#scipy.special.rel_entr. Python implementation. dtypeNr)npemptyshaperloginf)rrresc2Zc1rf/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/onnxruntime/quantization/calibrate.pyrel_entrs2 rbaseaxiscCs|dus |dks Jd|dusJdt|tj}d|tj||dd}t|tj}t||\}}d|tj||dd}t||}tj||d}|dur]|t|}||jS) z Simplifeied version of entropy. Source: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.entropy.html. This avoids taking a dependency on scipy just for this function. Nrz0base={base} must be a positive number or `None`.z qk is None?T)r!keepdimsr!) rasarrayastypefloat32sumZbroadcast_arraysrrr)rrr r!Zvecsrrrentropy's   r*c@sLeZdZegdZegdZddZeddZeddZ d d Z d S) TensorData)avgstdlowesthighesthist hist_edgesbins)r,r-r.r/r1cKst||_|D]E\}}|tjvr td|dtjd|tjvrJt|ds6tdt |d||j t j t j fvrJtd|j d|t|||q dS)NzUnexpected value z not in .rUnexpected type z for k=zUnexpected dtype )listkeys_attrsitemsr+_allowed ValueError_floatshasattrtyperrZfloat16r'setattr)selfkwargskvrrr__init__Gs   zTensorData.__init__cC4t|dr t|dstdt|d|j|jfS)Nr.r/z0Attributes 'lowest' and/or 'highest' missing in r3)r<AttributeErrordirr.r/r?rrr range_valueS zTensorData.range_valuecCrD)Nr,r-z)Attributes 'avg' and/or 'std' missing in r3)r<rErFr,r-rGrrravg_stdYrIzTensorData.avg_stdcs$fddjD}jj|d<|S)Ncsi|]}|t|qSr)getattr).0rArGrr asz&TensorData.to_dict..CLS)r7 __class____name__r?datarrGrto_dict_s zTensorData.to_dictN) rP __module__ __qualname__ frozensetr9r;rCpropertyrHrJrSrrrrr+Cs     r+c@sjeZdZdeeeeefffddZddZ ddZ dd Z d d Z d d Z ddZddZddZdS) TensorsDatarRc Cs||_i|_|D]p\}}t|tstdt|dt|trf|tj kr;t |dkr;t |d|dd|j|<q t |dkrUt |d|d|d|dd |j|<q td |d d t |d |dt|t sutdt|d||j|<q dS)NzKeys must be strings not r3rr r.r/)r.r/r0r2zUnexpected tuple for rz , it has z elements: zValues must be TensorData not ) calibration_methodrRr8 isinstancestr TypeErrorr=tupleCalibrationMethodMinMaxlenr+)r?r^rRrArBrrrrCgs"   &"  zTensorsData.__init__ccs|jEdHdSNrRrGrrr__iter__yszTensorsData.__iter__cCs ||jvSrfrgr?keyrrr __contains__| zTensorsData.__contains__cCs |j|Srfrgrirrr __getitem__rlzTensorsData.__getitem__cCs(||jvr td|d||j|<dS)Nz)Only an existing tensor can be modified, z is not.)rR RuntimeError)r?rjvaluerrr __setitem__s zTensorsData.__setitem__cC |jSrf)rRr6rGrrrr6rlzTensorsData.keyscCrqrf)rRvaluesrGrrrrrrlzTensorsData.valuescCrqrf)rRr8rGrrrr8rlzTensorsData.itemscCs|jj|j|jd}|S)N)rNrRr^)rOrPrRr^rQrrrrSs zTensorsData.to_dictN)rPrTrUrr`rr+rrCrhrkrmrpr6rrr8rSrrrrrXfs rXc@seZdZdZdZdZdZdS)rcrr rYr\N)rPrTrUrdEntropy Percentile Distributionrrrrrcs rcc@sVeZdZeddZejdefddZddZ dd Z d d Z d e d e fddZ dS)CalibrationDataReadercCst|dr t|jp tS)Nget_next)r<callablerwNotImplemented)clssubclassrrr__subclasshook__sz&CalibrationDataReader.__subclasshook__rcCt)z9generate the input data dict for ONNXinferenceSession runNNotImplementedErrorrGrrrrwszCalibrationDataReader.get_nextcCs|SrfrrGrrrrhzCalibrationDataReader.__iter__cCs|}|dur t|Srf)rw StopIteration)r?resultrrr__next__szCalibrationDataReader.__next__cCr}rfr~rGrrr__len__rzCalibrationDataReader.__len__ start_index end_indexcCr}rfr~)r?rrrrr set_rangerzCalibrationDataReader.set_rangeN)rPrTrU classmethodr|abcabstractmethoddictrwrhrrintrrrrrrvs rv) metaclassc@seZdZ     ddeeefdeeefddZdgfd d Z d d Z d e fddZ ddZ ddZdefddZdefddZdS)CalibraterBaseNaugmented_model.onnxF model_pathop_types_to_calibratecCsnt|tr tt||_nt|trt||_ntd||_||_||_||_ ||_ d|_ d|_ dg|_ dS)a :param model_path: ONNX model to calibrate. It should be a model file path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param symmetric: make range of tensor symmetric (central point is 0). :param use_external_data_format: use external data format to store model which size is >= 2Gb. :param per_channel: whether to compute ranges per each channel. z model_path should be model path.NCPUExecutionProvider)r_r`rrmodelr:raugmented_model_path symmetricuse_external_data_format per_channelZ augment_model infer_sessionexecution_providers)r?rrrrrrrrrrCs    zCalibraterBase.__init__rcCs||_|dS)zz reset the execution providers to execute the collect_data. It triggers to re-creating inference session. N)rcreate_inference_session)r?rrrrset_execution_providerss z&CalibraterBase.set_execution_providerscCs,t}tjj|_tj|j||jd|_dS)z9 create an OnnxRuntime InferenceSession. ) sess_options providersN) onnxruntimeZSessionOptionsZGraphOptimizationLevelZORT_DISABLE_ALLZgraph_optimization_levelZInferenceSessionrrr)r?rrrrrs  z'CalibraterBase.create_inference_sessionrc Csdd|jjD}|dd|jjD|dd|jjDdd|jjD}t}tjtj h}|jj D]4}|j rB|j |j vrkt |j|jD] }||vrj||}|jdrj|jjj|vrj||vrj||qJq7||fS) z select input/output tensors of candidate nodes to calibrate. returns: tensors (set): set of tensor name. value_infos (dict): tensor name to value info. cSi|]}|j|qSrnamerLvirrrrMz>CalibraterBase.select_tensors_to_calibrate..cSrrr)rLZotrrrrMrcSrrr)rLitrrrrMrcSh|]}|jqSrr)rLinitrrr z=CalibraterBase.select_tensors_to_calibrate.. tensor_typeN)graph value_infoupdateoutputinput initializersetr FLOATZFLOAT16noderop_type itertoolschainr=ZHasFieldr elem_typeadd) r?r value_infosrtensors_to_calibrateZtensor_type_to_calibrater tensor_namerrrrselect_tensors_to_calibrates(     z*CalibraterBase.select_tensors_to_calibratecC|jS)zP return: augmented onnx model. Call after calling augment_graph N)rrGrrrget_augment_modelsz CalibraterBase.get_augment_modelcCr})z abstract method: augment the input model to prepare for collecting data. It will: 1. augment the model to be able to collect desired statistics data 2. save augmented model to augmented_model_paths Nr~rGrrr augment_graphszCalibraterBase.augment_graph data_readercCr})z abstract method: collect the tensors that will be used for range computation. It can be called multiple times. Nr~)r?rrrr collect_datazCalibraterBase.collect_datarcCr})ze abstract method: compute data based on the calibration method stored in TensorsData Nr~rGrrr compute_data"rzCalibraterBase.compute_data)NrFFF)rPrTrUrr`rrrrCrrr rrrrvrrXrrrrrrs$   " rcs|eZdZ        ddeeefdeeeffdd Zd d Z d d Z d e fddZ ddZ defddZZS)MinMaxCalibraterNrF{Gz?rrc s|tj|||||| dg|_d|_t|jjj|_dd|jjjD|_ ||_ |r6|dks2|dkr6t d||_ ||_ dS)aw :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param symmetric: make range of tensor symmetric (central point is 0). :param use_external_data_format: use external data format to store model which size is >= 2Gb :param moving_average: compute the moving average of the minimum and maximum values instead of the global minimum and maximum. :param averaging_constant: constant smoothing factor to use when computing the moving average. :param max_intermediate_outputs: maximum number of intermediate outputs before an intermediate range is computed. :param per_channel: whether to compute ranges per each channel. )rrrrrNcSrrrrLrrrrrLrz,MinMaxCalibrater.__init__..rr z;Invalid averaging constant, which should not be < 0 or > 1.)superrCintermediate_outputscalibrate_tensors_rangererrrnum_model_outputsmodel_original_outputsmoving_averager:averaging_constantmax_intermediate_outputs) r?rrrrrrrrrrOrrrC*s" zMinMaxCalibrater.__init__csj\}}ttttjdgtj d}jj j |ddfdd}|D] }||d||dq1t jjjjd d S) z Adds ReduceMin and ReduceMax nodes to all quantization_candidates op type nodes in model and ensures their outputs are stored as part of the graph output :return: augmented ONNX model rcSs6|jD]}tj||jr|jSqtd|d)Nz&Model does not contain a version for 'z'.) opset_importonnxZdefsZhasdomainversionrn)rrrrrrget_op_version^s  z6MinMaxCalibrater.augment_graph..get_op_versionc sd}|d|}|d}tjj||g|g||d}tjjd|g|g|d}ddjjjD}|d djjjD|d djjjD||vrV||j j j }nt d |d j rt||j j jj} d gtd| } |jdkr|jtd| n tt} ttj| tjd| } |j| jjj| jjj||gjjjt ||dgdS)Nr _Z_Reshape)r#rZReshape)inputsoutputsrcSrrrrrrrrMwrzNMinMaxCalibrater.augment_graph..add_reduce_min_max..cSrrr)rLorrrrMxrcSrrrrLirrrrMyrz'Unable to guess tensor type for tensor zE, running shape inference before quantization may resolve this issue.rrYZaxesr)!rr Z make_noderrrrrrr=rrr:rrerZdimrange attributeappendZmake_attributer`uuiduuid4r from_arrayrarrayint64rrextendZmake_tensor_value_info) rZreduce_op_namer#Z reduce_outputintermediate_outputZ reduce_nodeZ reshape_noderZ onnx_typeZ tensor_rankZ reduced_axesZreduce_axes_nameZ reduce_axesrZreshape_shape_namer?rradd_reduce_min_maxds>     z:MinMaxCalibrater.augment_graph..add_reduce_min_maxZ ReduceMinZ ReduceMaxZsave_as_external_dataN)rrr`rrr rrrrrrrrsaverr)r?ZtensorsrZ reshape_shapertensorrrrrSs .   zMinMaxCalibrater.augment_graphcC g|_dSrfrrGrrrclear_collected_datarlz%MinMaxCalibrater.clear_collected_datarcCs |}|sq%|j|jd||jdur$t|j|jkr$|qt|jdkr5|jdur5t d| }t |t sHt dt|d|dS)NTrNo data is collected.z+compute_data must return a TensorsData not r3)rwrrrrunrrerrr:rr_rXrar=)r?rrtrrrrs"   zMinMaxCalibrater.collect_datac Cs|s|S|D]o\}}t|tr|jd}|jd}n|\}}t||tr6||jd}||jd}n||\}}|jrR||j||} ||j||} n t||} t||} t|tsht||trqt| | d||<q| | f||<q|S)Nrr rZ)r8r_r+rHrrminmax) r?Z old_rangeZ new_rangerjroold_minold_maxnew_minnew_max min_value max_valuerrr merge_ranges(      zMinMaxCalibrater.merge_rangerc stjdkr jSfddttjdDfddjD}i|D]}|D] \}}|g|q.q(jdfddtdtdD}fdd D}g}tdtdD]\}jrt j ||dd } t j ||d dd } nt j ||dd } t j ||d dd } j rt j t | t | gdd } |t| | gqg|t| | gqgttjtt||} jr݈j| _jS| _jS) z Compute the min-max range of tensor :return: dictionary mapping: {added node names: (ReduceMin, ReduceMax) pairs } rcsg|] }j|jqSr)r get_outputsrrrGrr sz1MinMaxCalibrater.compute_data..cg|] }tt|qSrrziprLr output_namesrrrNcsg|] }|ddqS)rr) rpartitionr)added_output_namesrrrsrYcs i|] }|jvr||qSr)rr)merged_output_dictr?rrrMs  z1MinMaxCalibrater.compute_data..r$r )rerrrr8 setdefaultrrrrmeanrrrabsrbrXrcrdrrr) r?output_dicts_listdrArBZcalibrate_tensor_namesZmerged_added_output_dictpairsrZmin_value_arrayZmax_value_arrayZmax_absolute_valueZnew_calibrate_tensors_ranger)rrrr?rrsF    zMinMaxCalibrater.compute_data)NrFFFrNF)rPrTrUrr`rrrrCrrrvrrrXr __classcell__rrrrr)s&  )I!rcsveZdZ         dd eeefd eeeffd d Zd dZ ddZ de fddZ de fddZZS)HistogramCalibraterNrF percentile-X@samerrc svtj|||||dg|_d|_t|jjj|_dd|jjjD|_ d|_ ||_ ||_ ||_ | |_d|_| |_dS)a= :param model_path: ONNX model to calibrate. It is a model path. :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. :param scenario: see :class:`DistributionCalibrater` )rrrrNcSrrrrrrrr+rz/HistogramCalibrater.__init__..)rrCrrrerrrrr collectormethodnum_binsnum_quantized_binsrrscenario) r?rrrrrrrrrrrrrrCs$ zHistogramCalibrater.__init__cCsV||j\|_}|jD]}||jvr|jjj||q tj|j|j |j ddS)z make all quantization_candidates op type nodes as part of the graph output. :return: augmented ONNX model rN) rrrrrrrrrrr)r?rrrrrr4s   z!HistogramCalibrater.augment_graphcCrrfrrGrrrrDrlz(HistogramCalibrater.clear_collected_datarc sBddjD}ddjD |}|sqIjd|}g}t|D]\}}||vr<|t|q)||q)j|qt jdkrTt dfd djD}i|D]} | D] \} } | g| qhqbfd d D} j stjjjjjjd _ j | dS) zy Entropy Calibrator collects operators' tensors as well as generates tensor histogram for each operator. cSrrrrLZnode_argrrrrKrz3HistogramCalibrater.collect_data..cSsg|]}|jqSrrrrrrrLrz4HistogramCalibrater.collect_data..TNrrcrrrrrrrrbrcs i|] }|jvr||qSr)rr) merged_dictr?rrrMks z4HistogramCalibrater.collect_data..)rrrrrr)r get_inputsrrwr enumeratercopyrrer:r8rrHistogramCollectorrrrrrrcollectr) r?rZinput_names_setrrZ fixed_outputsZ output_indexrr r rArBZclean_merged_dictr)rrr?rrGsH      z HistogramCalibrater.collect_datarcCsh|jstdt|trtj}nt|trtj}nt|tr"tj }n t dt |dt ||j S)z Compute the min-max range of tensor :return: dictionary mapping: {tensor name: (min value, max value)} z9No collector created and can't generate calibration data.zUnknown calibrater z". This method must be overwritten.N)rr:r_EntropyCalibraterrcrsPercentileCalibraterrtDistributionCalibraterrurar=rXcompute_collection_result)r?calrrrrzs   z HistogramCalibrater.compute_data) NrFrFrrrr)rPrTrUrr`rrrrCrrrvrrXrr rrrrrs&  ,3rcsFeZdZ       d deeefdeeeffdd ZZ S) r NrFr*rrrc  tj||||||||ddS)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. )rrrrNrrC) r?rrrrrrrrrrrrC zEntropyCalibrater.__init__)NrFr*Frr rPrTrUrr`rrrrCr rrrrr   r csFeZdZ       d deeefdeeeffd d ZZ S) r!NrFrrrrrc r%)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. )rrrrNr&) r?rrrrrrrrrrrrCr'zPercentileCalibrater.__init__)NrFrFrrr(rrrrr!r)r!csDeZdZ      d deeefdeeeffd d ZZ S) r"NrF distributionrrrrc stj|||||||ddS)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param scenario: for float 8 only, if `scenario="same"`, the algorithm weights and float 8 follow the same distribution, if `scenario="p3"`, it assumes the weights follow a gaussian law and float 8 ~ X^3 where X is a gaussian law )rrrNr&)r?rrrrrrrrrrrCs zDistributionCalibrater.__init__)NrFr*rrr(rrrrr"s  r"c@s,eZdZdZejddZejddZdS)CalibrationDataCollectorzL Base class for collecting data for calibration-based quantization. cCr})z Generate informative data based on given data. name_to_arr : dict tensor name to NDArray data Nr~r? name_to_arrrrrrsz CalibrationDataCollector.collectcCr})z? Get the optimal result among collection data. Nr~rGrrrr#sz2CalibrationDataCollector.compute_collection_resultN)rPrTrU__doc__rrrr#rrrrr+s  r+c@sveZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ ddZ e dddZddZddZdS)ra` Collecting histogram for each tensor. Percentile and Entropy method are supported. ref: https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py ref: https://docs.nvidia.com/deeplearning/tensorrt/pytorch-quantization-toolkit/docs/_modules/ pytorch_quantization/calib/histogram.html cCs.i|_||_||_||_||_||_||_dSrf)histogram_dictrrrrrr)r?rrrrrrrrrrCs zHistogramCollector.__init__cCrrf)r/rGrrrget_histogram_dictsz%HistogramCollector.get_histogram_dictcCsHtd|jdvr||S|jdkr |jr||S||Std)Nz/Collecting tensor data and making histogram ...>r*r*rDOnly 'entropy', 'percentile' or 'distribution' methods are supported)printr collect_valuercollect_absolute_valuer:r,rrrrs     zHistogramCollector.collectcCsv|D]3\}}t|trD|D]}t|tjs$Jdt|d|qtdd|D}t|dks>Jd|d|t|}nt|tjsVt dt|d||}| }|j dkrlt |}t |}ntjd|jd}tjd|jd}t|}||jvrtj||jd \} } | |j} |jtjksJd | | ||f|j|<q|j|} | d } | d } t| d sJdt| t| d sJdt| | d}| d}t |}||dkr|d|d}t|d||||}t||f}tj||d \} } | |j} | dt||7<|jtjks)Jd | | t | |t | |f|j|<qdS)z5 Collect histogram on absolute value r4z for tensor=css|]}|jVqdSrfr)rLarrr 6sz.r z6The calibration expects only one element type but got rr)r2zMonly float32 or float16 is supported, every constant must be explicitly typedrYr\rz'old_min should be a numpy array but is rN)r8r_r5rndarrayr=rrer%r:flattensizerrrrabsoluter/ histogramrr&float64r<ZarangeZhstack)r?r-rdata_arrZarrZdtypesZ data_arr_nprrr0r1 old_histogramrrold_histold_hist_edgesZ temp_amaxwidthZ new_bin_edgesrrrr4.sb &               z)HistogramCollector.collect_absolute_valuec Cs|D]k\}}t|}|}|jdkr!t|}t|}ntjd|jd}tjd|jd}tjtt |t ||jd}||j vrW|j |}| ||||||j |<qtj ||j | |fd\}} || |||f|j |<qdS)z1 Collect histogram on real value rrrN)r8rr%r8r9rrrrr r/merge_histogramr;r) r?r-rr=rr thresholdr>r0r1rrrr3hs,         z HistogramCollector.collect_valuecCs|\}}}} } || kr)tj|t|| | fd\} } | ||t||t| || fS| dkrAtj|t|| |fd\} }| |7} n9t|}d| |}t|| |d}|d|}||| }tj||| |fd\} }| ||||7<| |t||t| ||fS)NrBrrYr )rr;rerrr)r?r>r=rrZ new_thresholdr?r@rrZ old_thresholdZnew_histrr0r1Z old_num_binsZ old_strideZhalf_increased_binsZ new_num_binsrrrrCs2    z"HistogramCollector.merge_histogramcCsl|jr t|jdkrtdtd|jd|jdkr |S|jdkr)|S|jdkr2|Std) Nrz=Histogram has not been collected. Please run collect() first.z0Finding optimal threshold for each tensor using z algorithm ...r*rr*r1)r/rer:r2rcompute_entropycompute_percentilecompute_distributionrGrrrr#s   z,HistogramCollector.compute_collection_resultcCs|jdks |jdkrtd|j}|j}i}tdt|td|jtdd|d|d |D]\}}|d}|d }|}t ||} |j rrt | |d} tj || |j d  tj || |j d f||<n*d|d } t | d | } t | | } tj || |j d tj || |j d f||<|d} |d}||d| kr| ||d f||<||d |kr||d|f||<g|||ddR||<tjdddvrt||q8|S)Nrdz     z'HistogramCollector.compute_distributioncs|d}|d}|j}|d}|d}|djt||d}fddt|jD} t||ddD]} || } t|| d|} || || f| | |<t|| | } | }t|d| }t|| d}|d|7<|d|7<|dk tj }tj|tj d}| j|}t|D]}||}||}t| ||||<q|dt| ||d7<tj|jtj d}t|D] }||}||}t|||}|dkr||||||<qt |}t |}|dus|durtj tj d}n tj t||d}||| |<q5t|}| |}|d}|d }|d|kr2||df}|d|kr?|d|f}t|dd sIJt|dd sSJ|S) aFGiven a dataset, find the optimal threshold for quantizing it. The reference distribution is `q`, and the candidate distribution is `p`. `q` is a truncated version of the original distribution. Ref: http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf rr rYcs(g|]}tjddtjddfqS)rr)rrrrrrrEs(z.Nrrr\r)r9rrzerosrrrdeepcopyr(r&rrrrr*Zargminr<)r?r;rr0r1rZzero_bin_indexZnum_half_quantized_binZ kl_divergenceZ thresholdsrrrZsliced_distributionpZleft_outliers_countZright_outliers_countZnonzerosZquantized_binsZnum_merged_binsindexstartendqZnormdivZmin_kl_divergence_idxrVrrrrrrU7sf        z(HistogramCollector.get_entropy_thresholdN)r )rPrTrUr.rCr0rr4r3rCr#rFrE staticmethodrZrGrUrrrrr s :  .  (rrFrrc CsNd}|tjkr3|dd}|dd}|dd} |dd} |dd} t||||||| | | d }na|tjkrV|d d } |d d } |dd}t|||||| | d }n>|tjkry|d d } |dd}|dd}t|||||| |d}n|tjkr|d d } |dd}t ||||| |d}|r| | |St d|)NrFrrrrr)rrrrrrrrr)rrrrrrrT)rrrrrr)rrrzUnsupported calibration method ) rcrdrRrrsr rtr!rur"rrr:)rrrZcalibrate_methodrZ extra_optionsZ calibratorrrrrrrrrrrrrcreate_calibratorsx               rd)Nr)0rrrrPrenumrpathlibrtypingrrrrrZnumpyrrr r r r rZ quant_utilsrrrr7rfloatrr*r+rXrcABCMetarvrrrr r!r"r+rrdr`rdrrrrsl   #4n_""#