o
    4ѹg %                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
m  mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ eeZeejed
krbd dlmZ nedddZe ddG dd dejj!Z"dS )    N)partial)	OmegaConf)contextmanager)LooseVersion)tables)AltBlock)AudioEncoder)load_audio_text_image_videoz1.6.0)autocastTc                 c   s    d V  d S )N )enabledr   r   c/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/funasr/models/emotion2vec/model.pyr
      s   
r
   Zmodel_classesEmotion2vecc                       sh   e Zd ZdZ fddZ									dddZ	dd	d
Z				ddefddZdd Z	  Z
S )r   z
    Author: Ziyang Ma, Zhisheng Zheng, Jiaxin Ye, Jinchao Li, Zhifu Gao, Shiliang Zhang, Xie Chen
    emotion2vec: Self-Supervised Pre-Training for Speech Emotion Representation
    https://arxiv.org/abs/2312.15185
    c                    sj  t    t|d   | _ttjj 	d 	ddd fdd	i | _
tj | _t jj 	d 	d| j
}|| jd	< d | _ 	d
| _ 	d| _ 	d| _tj 	d| _t 	d 	d 	dtjfddt 	dD | _d | _ 	dr 	d| _|	dd}d | _|dkrtj 	d|| _d S d S )NZ
model_confZnorm_epsZnorm_affine)ZepsZelementwise_affinec                    sp   t |d u r
 dn||d u r dn| dd d d d d|  d	 d
 dS )N	embed_dimZ	num_headsZ	mlp_ratioTZencoder_dropoutZattention_dropoutZactivation_dropoutpost_mlp_droplayer_norm_firstZend_of_block_targets)	Zqkv_biasZdropZ	attn_dropZmlp_dropr   	drop_pathZ
norm_layerr   Zffn_targets)r   get)r   dimZheads)cfgmake_layer_normr   r   
make_block4   s   
z(Emotion2vec.__init__.<locals>.make_blockr   r   AUDIOaverage_top_k_layers	loss_beta
loss_scaledropout_inputZstart_drop_path_rateZend_drop_path_ratedepthc                    s   g | ]} | qS r   r   ).0i)dprr   r   r   
<listcomp>]   s    z(Emotion2vec.__init__.<locals>.<listcomp>
vocab_sizer   )NN)super__init__r   creater   r   torchnnZ	LayerNormr   Zalibi_biasesZ
ModuleDictmodality_encodersr   Z
modalitiesaudioZemar   r   r   ZDropoutr   npZlinspaceZ
ModuleListrangeblocksnormprojZLinear)selfkwargsencr"   	__class__)r   r    r   r   r   r%   *   sF   

(
zEmotion2vec.__init__NTFc              	   K   s  | j d }d }||||| p||s| jdnd||
d}|d }|d }|d }|dd }|d	d }| jd ur?| |}g }t| jD ]M\}}| jrb| jd
ddksbtj | jd
dkr|}|d ur|d ur|	ddkrw|| n|
d}||| }||||d\}}|r|| qF| jd ur| |}|r|	r|d d |jjd f }|d ur|d d |jjd f }||||dS d S )Nr   clone_batch   )Zremove_maskedr5   
mask_seedsprecomputed_maskxencoder_maskpadding_mask
alibi_biasalibi_scaleZ	layerdropr   )r;   r<   )r9   r;   layer_resultsmask)r)   r   r   r   	enumerater-   Ztrainingr+   randomsizesqueezeZtype_asappendr.   Zmodality_cfgZnum_extra_tokens)r0   sourcetargetidmoder;   r?   features_onlyZforce_remove_maskedremove_extra_tokensr8   r1   Zfeature_extractorr7   Zextractor_outr9   r:   Zmasked_padding_maskZmasked_alibi_biasr=   r>   r   ZblkabZscalelrr   r   r   forwardh   sj   



 



zEmotion2vec.forwardc                 C   s   | j ||||d|d}|S )NT)rH   r;   r?   rI   rJ   )rM   )r0   rE   rH   r;   r?   rJ   resr   r   r   extract_features   s   zEmotion2vec.extract_featureskeyc              
      sD  | dd}| dd}| jd u rd}i }	t }
t|d| dd| dd|d	}t }||
 d
|	d< t|d | dd |	d< g }| d}|rTtj|dd t|D ]\}}|j	|d d}| j
jrot||j}|dd}| j|d d}|d }|d d  }|dkr|}n|dkrtj|dd}|r|rttj|d|| | |d ur|jng }g  | jr|jdd}| |}t|D ]\}}|drtj n|d d |f |d d |f< qtj|dd}|d   dd |D } fddt|D }|| ||d}|r||d< | | qX||	fS ) NgranularityZ	utteranceextract_embeddingTi>  fs	data_typeZsound)rS   Zaudio_fsrT   	tokenizerz0.3fZ	load_datar   Zbatch_data_time
output_dir)exist_okdevice)rX   r6   r#   )r;   r9   frame)Zaxisz{}.npy)r   unusec                 S   s   g | ]	}| d s|qS rZ   
startswith)r   lbr   r   r   r!     s    z)Emotion2vec.inference.<locals>.<listcomp>c                    s"   g | ]\}}| d s | qS r[   r\   )r   idxr^   scoresr   r   r!     s   " )rP   labelsra   feats)!r   r/   timeperf_counterr	   lenosmakedirsr@   tor   	normalizeFZ
layer_normshapeviewrO   rC   cpunumpyr+   meansavepathjoinformatZ
token_listr]   infr'   ZsoftmaxtolistrD   )r0   Zdata_inZdata_lengthsrP   rU   Zfrontendr1   rQ   rR   Z	meta_dataZtime1Zaudio_sample_listZtime2resultsrV   r   ZwavrE   rc   r9   rb   r_   ZlabZselect_labelZselect_scoreZresult_ir   r`   r   	inference   sd   



 
0zEmotion2vec.inferencec                 K   s"   ddl m} |dd| i|}|S )Nr6   )export_rebuild_modelmodelr   )Zexport_metary   )r0   r1   ry   modelsr   r   r   export  s   zEmotion2vec.export)	NNNNTFFTN)NNFT)NNNN)__name__
__module____qualname____doc__r%   rM   rO   listrx   r|   __classcell__r   r   r3   r   r   "   s.    A
M

L)T)#rg   rd   r'   loggingro   r+   	functoolsr   Z	omegaconfr   Ztorch.nn.functionalr(   Z
functionalrk   
contextlibr   Zdistutils.versionr   Zfunasr.registerr   Z!funasr.models.emotion2vec.modulesr   Zfunasr.models.emotion2vec.audior   Zfunasr.utils.load_utilsr	   	getLoggerr}   logger__version__Ztorch.cuda.ampr
   registerModuler   r   r   r   r   <module>   s*   

