o
    4ѹgq                     @   sV  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z
d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl$m&Z& zd dl'm(Z(m)Z)m*Z* d dl+m,Z, W n   Y dddZ-G dd dZ.dS )    N)tqdm)
DictConfig
ListConfig)deep_update)tables)
load_bytes)download_from_url)timestamp_sentence)timestamp_sentence_en)download_model)slice_padding_audio_samples)	merge_vad)load_audio_text_image_video)set_all_random_seed)load_pretrained_model)export_utils)misc)sv_chunkpostprocessdistribute_spk)ClusterBackendc                    s  g }g }g d}t jt j  t| tr!| ds| dr!t| } t| trtj	| rtj
| \}}| }||v rt| ddg}	|	D ]Y}
dd fdd	td
D  }| drrt|
 }|d }d|v ro|d n|}n"|
 jdd}t|dkr|d n|d }t|dkr|d n|}|| || qEW d   ||fS 1 sw   Y  ||fS |du rt| }| g}|g}||fS t| ttfr?|durt|ttfrg }t| |D ]\}}t||d\}}|| qg }t| D ]>}|| q| }g }| D ]0}t|trtj	|rt|}n|du r4dd fdd	td
D  }|| q
||fS t| trIt| } |du r^dd fdd	td
D  }| g}|g}||fS ) )z.scpz.txtz.json.jsonlz.textzhttp://zhttps://zutf-8)encodingZ	rand_key_ c                 3       | ]}t  V  qd S Nrandomchoice.0_chars Z/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/funasr/auto/auto_model.py	<genexpr>;       z(prepare_data_iterator.<locals>.<genexpr>   r   sourcekey   )maxsplitr   N)data_in	data_typec                 3   r   r   r   r    r#   r%   r&   r'   a   r(   c                 3   r   r   r   r    r#   r%   r&   r'   h   r(   )stringascii_lettersdigits
isinstancestr
startswithr   ospathexistssplitextloweropenjoinrangeendswithjsonloadsstripsplitlenappendr   Z"extract_filename_without_extensionlisttuplezipprepare_data_iteratorbytesr   )r.   	input_lenr/   r+   	data_listkey_listfilelistr"   Zfile_extensionZfinlinelinesdataZdata_list_tmpZ	data_in_iZdata_type_iZdata_list_iitemZdata_ir%   r#   r&   rH   (   s|   
 


33
 

 

 rH   c                   @   sP   e Zd Zdd Zedd Zdd Zddd	Zdd
dZdddZ	dddZ
dS )	AutoModelc                 K   sf  zddl m} ||ddd W n   Y tt|dd }tj|d | jdi |\}}|d	d }|d
i d u rAi n|d
i }|d urltd ||d< |dd|d< |d |d< | jdi |\}}|dd }|di d u r|i n|di }|d urtd ||d< |dd|d< |d |d< | jdi |\}}|dd }	|di d u ri n|di }
|
di d u ri n|
di }|	d urtd |	|
d< |dd|
d< |d |
d< | jdi |
\}	}
t	di |
|d | _|dd}|dvrtd || _|| _|| _|| _|| _|| _|| _|	| _|
| _|d| _d S ) Nr   )check_for_updateZdisable_updateF)disable	log_levelINFO)level	vad_model
vad_kwargszBuilding VAD model.modelZvad_model_revisionZmasterZmodel_revisiondevice
punc_modelpunc_kwargszBuilding punc model.Zpunc_model_revision	spk_model
spk_kwargs	cb_kwargszBuilding SPK model.Zspk_model_revisionspk_modepunc_segment)defaultvad_segmentrb   z@spk_mode should be one of default, vad_segment and punc_segment.
model_pathr%   )Zfunasr.utils.version_checkerrS   getgetattrloggingupperbasicConfigbuild_modelinfor   tocb_modelerrorra   kwargsrZ   rX   rY   r\   r]   r^   r_   re   )selfrp   rS   rU   rZ   rX   rY   r\   r]   r^   r_   r`   ra   r%   r%   r&   __init__q   s^    
 
  



zAutoModel.__init__c               
   K   s  d| v sJ d| vrt d| dd td-i | } t| dd | dd	}tj r8| d
ddkr>d}d| d< || d< t	| dd | dd }|| d< d| d< |d urt
|trh|dn|}| di }g }g }g }| dg }| dg }	t
|tttfs|gt| }t|D ]h\}
}tj|}||
 }t|dkr||
 |d< t|	dkr|	|
 |d< |d-i |}|| t|dr|jnd }t|dr| n|}d}|d urt|}|dkrt|dr| }|| || qt|dkr|d }|d }|d }|| d< || d< || d< | dd }d | d< |d urKtj|}|d-i | di }t|drG| nd | d< || d< tj| d }|d useJ | d  di }t|| di  t||  |d-i |}| d d }|d urtj|rt d!|  t ||| d"d#| d$d | d%g | d&d d' nt!d(|  | d)d*r|"tj# n| d+d*r|"tj$ |"| | d,d#st!  || fS ).NrZ   
model_confz"download models from model hub: {}Zhubmsseedr   r[   cudaZngpur,   cpu
batch_sizeZncpu   	tokenizer
vocab_size,tokenizer_conftoken_lists	seg_dicts
token_listZseg_dict	get_vocabget_vocab_sizefrontendZ
input_sizeZfrontend_confoutput_sizez is not registered
init_paramzLoading pretrained params from ignore_init_mismatchT
oss_bucket	scope_mapexcludes)rZ   r7   r   r   r   r   z#error, init_param does not exist!: Zfp16FZbf16Zdisable_logr%   )%rh   rl   formatrf   r   r   torchrv   Zis_availableZset_num_threadsr3   r4   rB   rE   rF   r   rC   	enumerater   Ztokenizer_classesrD   hasattrr   r   r   Zfrontend_classesr   Zmodel_classesr   r6   r7   r8   r   printrm   Zfloat16Zbfloat16)rp   r[   rz   Z
tokenizersZtokenizers_confZtokenizers_buildZvocab_sizesr   Ztoken_list_filesr   iZtokenizer_classr~   r   r|   r   Zfrontend_classZmodel_classrs   rZ   r   r%   r%   r&   rk      s   









	
zAutoModel.build_modelc                 O   s(   | j }t|| | jg ||R  }|S r   )rp   r   rZ   )rq   argscfgrp   resr%   r%   r&   __call__%  s   
zAutoModel.__call__Nc                 K   s6   | j d u r| j|fd|i|S | j|fd|i|S )NrJ   )rX   	inferenceinference_with_vad)rq   inputrJ   r   r%   r%   r&   generate+  s   
zAutoModel.generatec              	   K   sx  |d u r| j n|}d|v r|d t|| |d u r| jn|}|  |dd}t|||dd |d\}}	i }
g }t|	}| j dd}|sPtd|d	d
nd }d}d}t	d||D ]}t
||| }|	|| }||| }||d}|| dkr|dd dkr|d |d< ||d< t }t 4 |jdi ||}t|ttfrt|dkr|d nddig}t|dkr|d ni }W d    n1 sw   Y  t }|| |dd}|| }|dd|
d< |dd|
d< |d|
d< t| |
d< || d|
d< |
 d}|r|||  || ||7 }||7 }q\|r5|d|| d tj  |S )Ncacherx   r,   r/   rJ   r/   r+   disable_pbarFZblueTZcolourtotalZdynamic_ncolsg        r   )r.   r+   Zfbankr.   Zdata_lengthstextr   batch_data_timer{   Z	load_dataZextract_feat0.3fZforwardZrtfz, 	rtf_avg: r%   )rp   popr   rZ   evalrf   rH   rC   r   r=   mintimeperf_counterr   no_gradr   r3   rE   rF   extendupdateset_descriptionrv   Zempty_cache)rq   r   rJ   rZ   rp   r+   r   rx   rL   rK   Zspeed_statsZasr_result_listZnum_samplesr   ZpbarZtime_speech_totalZtime_escape_totalbeg_idxend_idxZ
data_batchZ	key_batchbatchZtime1r   resultsZ	meta_dataZtime2r   Ztime_escapedescriptionr%   r%   r&   r   2  sj   









zAutoModel.inferencec           >   
      s  | j }t| j| t }| j|f|| j| jd|}t }|ddrBtt|D ]}t	|| d |ddd || d< q,| j
}	t|| tt|dd	d d
}
t|ddd }|
|d< t|||dd d\}}g }d}t }|ddstdt|ddnd }tt|D ]}|| d }|| d  || }t|d dr|d jnd}t|||ddd}t|}t } fddt|D }t|dd d}g }t|s||dg d  td!| qt|d"krt|d" d"krt|
|d" d" d
 |d" d" d"  }
|d# d$krd"}
d"}t }|d }||7 }g }d"}d
} ttd"|D ]\}!}"||! d" d
 ||! d" d"  }#t||#|!d
 |  }$|!|d
 k rr|#|k rr|$|
k rrt||#}| d
7 } q7t|||||  \}%}&| j|%fd |	|d|}'| jd urtt|%D ]N}(|||  |( d" d" d% |||  |( d" d
 d% t|%|( gg})t|)}*||* d&d |*D }+| j|+fd | j|d|},|,d" d' |'|( d'< q| }| d
7 } |#}t|'d
k rq7||' q7t||kr||dg d  td(| qd"g| }-t|D ]}!||! d
 }.||! |-|.< q i }/t|D ]}!|-|!  D ]\}0}1|0d)r|0|/vrQg |/|0< |-|! |0 D ]}2|2d"   |! d" 7  < |2d
   |! d" 7  < qW|/|0 |-|! |0  q>|0d'kr|0|/vr|-|! |0 |/|0< q>t j!|/|0 |-|! |0 gd"d*|/|0< q>d+|0v r|0|/vr|-|! |0 |/|0< q>|/|0  d,|-|! |0  7  < q>|0|/vr|-|! |0 |/|0< q>|/|0  |-|! |0 7  < q>q6t|/d+ " sq|d-d}3d }4| j#d ur.t| j$| | j|/d+ f| j#| j$d.|}5t%%|/d+ }4|3r&|4|/d/< |5d" d+ |/d+< | jd ur|d0dr|4d u rEt&d1 t|d2d d}|/d' }6| j'|6( |d3d d4}7t)|d |7|6( }8| j*d5krg }9t+|- D ]U\}:};d)|:vrt&d6 |9|;d" |;d
 |:d+ |:d) d7 qt| j*d8krd)|/vrt&d9 |d:drt,|5d" d; |/d) |4|3d<}9nt-|5d" d; |/d) |4|3d<}9t.|9|8 |9|/d=< n;|d>drt|/d+ " sg }9n$|d:drt,|5d" d; |/d) |4|3d<}9nt-|5d" d; |/d) |4|3d<}9|9|/d=< d'|/v r|/d'= ||/d< ||/ t }<|<| }=|rC|/d
 |0d?|=| d@dA|dBdC|=d@ q|S )DN)rJ   rZ   rp   r   FvalueZmerge_length_s   i  Zbatch_size_si,  r,   Zbatch_size_threshold_s<   rx   r/   )rJ   r/   gư>r   ZredTr   r+   r   fsi>  )r   Zaudio_fsc                    s   g | ]} | |fqS r%   r%   r!   r   Zvadsegmentsr%   r&   
<listcomp>      z0AutoModel.inference_with_vad.<locals>.<listcomp>c                 S   s   | d d | d d  S )Nr   r,   r%   xr%   r%   r&   <lambda>  r   z.AutoModel.inference_with_vad.<locals>.<lambda>)r+   r   )r+   r   	timestampzdecoding, utt: {}, empty speechr   r[   rw   g     @@c                 S   s   g | ]}|d  qS )   r%   r   r%   r%   r&   r     s    spk_embeddingzdecoding, utt: {}, empty resultr   )Zdimr   r   return_raw_text)rZ   rp   raw_textZreturn_spk_resz3Missing punc_model, which is required by spk_model.c                 S   s   | d S )Nr   r%   r   r%   r%   r&   r   $  s    Zpreset_spk_num)Z
oracle_numrd   aC  Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch'                                            and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'                                           can predict timestamp, and speaker diarization relies on timestamps.)startendZsentencer   rb   a;  Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch'                                        and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'                                       can predict timestamp, and speaker diarization relies on timestamps.Zen_post_procZ
punc_array)r   Zsentence_infoZsentence_timestampr   r   z, time_speech: z 0.3fz, time_escape: )1rp   r   rY   r   r   rX   rf   r=   rC   r   rZ   maxintrH   r   r   r   r   sortedrD   rh   rl   r   r   r   r^   nparrayr   r   itemsr5   r   catrA   r\   r]   copyro   rn   rw   r   ra   rG   r
   r	   r   r   r   )>rq   r   rJ   r   rp   Zbeg_vadr   Zend_vadr   rZ   rx   Zbatch_size_threshold_msrL   rK   Zresults_ret_listZtime_speech_total_all_samplesZ	beg_totalZ
pbar_totalr+   Zinput_ir   ZspeechZspeech_lengthsnZdata_with_indexZsorted_dataZresults_sortedr   Zbeg_asr_totalZtime_speech_total_per_sampleZall_segmentsZmax_len_in_batchr   jr"   Zsample_lengthZpotential_batch_lengthZspeech_jZspeech_lengths_jr   _bZvad_segmentssegmentsZspeech_bZspk_resZrestored_dataindexresultkvtr   r   Zpunc_resr   labelsZ	sv_outputZsentence_listrestZ
vadsegmentZend_asr_totalZtime_escape_total_per_sampler%   r   r&   r   t  s  



 & 


	



&

 













zAutoModel.inference_with_vadc           
      K   s   | dd}| jj|d}| j}t|| ||d< |d= |  | dd}t|d| dddd	\}}t  t	j
d||d
|}	W d   |	S 1 sOw   Y  |	S )z

        :param input:
        :param type:
        :param quantize:
        :param fallback_num:
        :param calib_num:
        :param opset_version:
        :param cfg:
        :return:
        r[   rw   )r[   rZ   typeZonnxNr/   r   )rZ   r.   r%   )rf   rZ   rm   rp   r   r   rH   r   r   r   export)
rq   r   r   r[   rZ   rp   r   rL   rK   Z
export_dirr%   r%   r&   r   |  s"   



zAutoModel.exportr   )NNNN)__name__
__module____qualname__rr   staticmethodrk   r   r   r   r   r   r%   r%   r%   r&   rR   o   s    <
w


B  
rR   )NNN)/r?   r   r   r   r   r0   rh   os.pathr6   Znumpyr   r   Z	omegaconfr   r   Zfunasr.utils.miscr   Zfunasr.registerr   Zfunasr.utils.load_utilsr   Zfunasr.download.filer   Zfunasr.utils.timestamp_toolsr	   r
   Z'funasr.download.download_model_from_hubr   Zfunasr.utils.vad_utilsr   r   r   Z&funasr.train_utils.set_all_random_seedr   Z(funasr.train_utils.load_pretrained_modelr   Zfunasr.utilsr   r   Zfunasr.models.campplus.utilsr   r   r   Z&funasr.models.campplus.cluster_backendr   rH   rR   r%   r%   r%   r&   <module>   s@   
G