o
    4ѹgIQ                  	   @   sn  d dl Z d dlZd dlZd dlZd dlmZ d dlm  mZ	 d dl
mZ d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d d	lmZ e eZed
g dZedg dZG dd dejZ dd Z!dee fddZ"dej#dedej#fddZ$dej#dedej#fddZ%		d*de&de&d e&d!e'fd"d#Z(		d*d$d%Z)d&d' Z*d(d) Z+dS )+    N)
namedtuple)	dataclass)partial)MISSINGII)OptionalCallable)compute_mask_indices)GradMultiply)	index_putMaskSeedseedupdateidsMaskInfo
x_unmaskedmaskids_restoreids_keepc                       s  e Zd Zdedejdejdeej deej dejdejdeeeeeege	j
f  f fd	d
Zdd Zdd ZdefddZdd Z			d*dedee	j
 fddZ			d*dedededee	j
 fddZdd Zd ee fd!d"Zd+d#d$Zd%d& Zd,d(d)Z  ZS )-ModalitySpecificEncoder	embed_dimlocal_encoderproject_featuresfixed_positional_encoderrelative_positional_encodercontext_encoderdecoderget_alibi_biasc
                    sx  t    || _|| _|| _|| _|| _|| _|| _|j	r|	nd | _
| jj| _d | _|jdkr]ttd|j|| _|jsFtj| j n| jddkr]tj| jd d dd f  d | _| j
d urtjtj|jrr|j|j ndd|jr{| jjndddf|jtjd|jd| _|jr| j
d ur|j d usJ | j
d|j |jdtjdd}
t|
| _!t"t#| j!d| _
d S d S d S )	Nr      )dtype)Zrequires_grad      ?cpu)
batch_size
time_stepsheadsscaler!   device)
alibi_bias)$super__init__modality_cfgr   r   r   r   r   r   Zuse_alibi_encoderr   local_grad_multextra_tokensnum_extra_tokensnn	ParametertorchzerosZinit_extra_token_zeroinitnormal_sizealibi_scalefullZlearned_alibi_scale_per_layerprenet_depthZmodel_depthZlearned_alibi_scale_per_headnum_alibi_headsfloatZlearned_alibi_scaleZlearned_alibiZalibi_max_posr)   r   _learned_alibi_bias)selfr,   r   r   r   r   r   r   r   r   r)   	__class__ b/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/funasr/models/emotion2vec/base.pyr+      sf   




z ModalitySpecificEncoder.__init__c                 C   s8   | d}||v r||   dkr|| d||< |S )Nz.alibi_scale   r   )dim	unsqueeze)r=   Z
state_dictnamekr@   r@   rA   upgrade_state_dict_namedh   s   
z0ModalitySpecificEncoder.upgrade_state_dict_namedc                 C   s   |S Nr@   )r=   xpadding_maskr@   r@   rA   convert_padding_masko      z,ModalitySpecificEncoder.convert_padding_mask	mask_infoc           	      C   s(  | j jj}|dkrtj||| jdd}| j j}|d urr|jjd |jd  | }|	|
d||
dd| j j}tj|d d |d f |gdd}tj|d|jd}| j jjrq| jd usaJ | |d }|||jd  }n
|d d |d f }| j jjr| jd usJ || |d  }||fS )Nr   T)trainingZinplacer    rC   rC   index)r,   r   Zinput_dropoutFZdropoutrN   r/   r   shape	new_emptyr6   r5   mask_noise_stdr2   catgatherZadd_positions_maskedr   r   rD   Zadd_positions_all)	r=   rI   rM   Zinp_dropZ	num_extraZ
num_maskedZmask_tokensZx_posr@   r@   rA   decoder_inputr   s2   
"

z%ModalitySpecificEncoder.decoder_inputc                 C   sv   | j dkr| j dkr| |}n$t| || j }nt  | |}W d    n1 s/w   Y  | |}|S )Nr   r"   )r-   r   r
   applyr2   Zno_gradr   )r=   featuresrI   r@   r@   rA   local_features   s   



z&ModalitySpecificEncoder.local_featuresr    Nclone_batch
mask_seedsc                    s(  |d ur
|  ||}|}|r|dkr| }|j\}	}
}|	}d }d }| jd ur/|| || }|r|dkr||d} d ur| fddt|d D }tdg|  	dd} j
}||d}|	d||| }|	d}t j j|d |d ur||d}| j|| | jd up| |d\}}| jd ur| |}|}|r|r|j}|d ur|t|| }|d ur| rt||}| sd }nd }n|d ur|| }d }| j}| jd ur$| j||
| jjtj|jd}|d ur|d}|ddkr||d| }d }|dkr||d}|d ur$|r$t||}| j d ur^| j d}tj!| j "|ddd|gdd	}|d urOt#$||df}|d ur^t#$||d|df}| %||||d urp|d | jj& nd }|||||d ur|ddkr|| jj&d  |d
S ||d
S )Nr    r   c                    s"   g | ]}t t j|fd  qS )g    _B)inthashr   ).0indr_   r@   rA   
<listcomp>   s    zCModalitySpecificEncoder.contextualized_features.<locals>.<listcomp>rO   r   )	mask_seedr[   precomputed_mask)r$   r%   r&   r!   r(   rP   )rI   r]   rJ   r)   r7   Zencoder_mask)'rK   clonerT   r   Zrepeat_interleaveranger2   Ztensorlongviewr   tor   r   r   compute_maskr   r   gather_unmaskedanygather_unmasked_maskr7   r   r,   r:   Zfloat32r(   Z	clamp_minr6   Zsqueezetype_asmasked_alibir.   rW   expandrS   padr   r9   )r=   rI   rJ   r   remove_maskedr^   r_   rg   r]   Zorig_BZorig_T_Z
pre_mask_BrM   Zx_posZ
clone_hashidZmasked_padding_maskr)   r7   numr@   rd   rA   contextualized_features   s   











$

z/ModalitySpecificEncoder.contextualized_featuresr   ru   c           	   	   C   s    |  |}| |||||||S rH   )r]   ry   )	r=   r\   rJ   r   ru   r^   r_   rg   rI   r@   r@   rA   forward  s   

zModalitySpecificEncoder.forwardc                 C   s   d S rH   r@   )r=   r@   r@   rA   reset_parameters0  rL   z(ModalitySpecificEncoder.reset_parametersrf   c                 C   s0  |d ur|}|  ||}n|j\}}	}
| j}|j}|jd ur0|jdkr0|j|k r0tj|j|}|dkr|jdkr@t	|||}nL| jj
rHd| }t||	f|||jdd|j|j|d ur]|jnd |d ure|jnd |d urm|jnd d}t|j|jd}| jj
rd| }|  ||}nd }|r| ||}||fS )Nr   r    T)Z	min_masksZrequire_same_masksmask_dropout	add_masksr   epochindicesr(   )make_maskinforT   r,   	mask_probZmask_prob_minnprandomuniformZmask_lengthrandom_maskingZinverse_maskr	   r|   r}   r   r   r   r2   
from_numpyrl   r(   
apply_mask)r=   rI   rJ   rf   r[   rg   r   rM   BTCcfgr   r@   r@   rA   rm   3  sN   
z$ModalitySpecificEncoder.compute_maskc                 C   s   |d u r|j \}}}n|\}}}|tj}|jdd}|jddddd|}||d   }	| jj	dkrF|	t
|t|	 | jj	 7 }	|d d d |	f }
|d urWd }n|
ddd|}
tj|d|
d}t||||
d}|S )Nr    rP   rO   r   rQ   r   )rT   rl   r2   Zuint8argsortrD   rs   sumr,   Zkeep_masked_pctroundr`   rX   r   )r=   rI   r   rT   r   r   Dids_shuffler   len_keepr   r   rM   r@   r@   rA   r   l  s*   
z%ModalitySpecificEncoder.make_maskinfoc                 C   s   | j }|j\}}}|d ur:|j}|jr |d||d  }n|  }|||	d
d|j}	t|||	}|jdkrat||fd |j|j}
t|
|jdd|d}
t||
d}|S )Nr    rO   r   )r,   rT   r   Zencoder_zero_maskrq   rD   r   itemrU   r6   r5   rV   r   Zmask_channel_probr	   Zmask_channel_lengthr2   r   rl   r(   rs   )r=   rI   rM   r   r   r   r   r   Z	num_masksmasksZmask_channelr@   r@   rA   r     s(   
 z"ModalitySpecificEncoder.apply_maskFc                 C   s   |sd | _ d S d S rH   )r   )r=   Zkeep_decoderr@   r@   rA   remove_pretraining_modules  s   
z2ModalitySpecificEncoder.remove_pretraining_modules)r    NNrH   )F)__name__
__module____qualname__r`   r0   Moduler   r   strr2   Tensorr+   rG   rK   r   rZ   r]   ry   boolrz   r{   r   rm   r   r   r   __classcell__r@   r@   r>   rA   r      sd    	
J 
 


9r   c                 C   s,   ||kr|S ||  }d||  }|||  S )Nr    r@   )startendZ	curr_stepZtotal_stepsrZpct_remainingr@   r@   rA   get_annealed_rate  s
   r   rf   c                 C   s(  | j \}}}t|d|  }d }|d ur2tt|j|j|j  fd }tj	| j
d}|| tj|||| j
d}	|	jdd}
|
jdd}|
d d d |f }|ddd|}tj| d|d}tj||g| j| j
d}d	|d d d |f< tj|d|d}|ddd|}t||||d
S )Nr    g    .Ar   )	generatorr(   rP   rO   rQ   r!   r(   r   r   )rT   r`   ra   r   r   r   r   r   r2   	Generatorr(   Zmanual_seedZrandr   rD   rs   rX   Zonesr!   r   )rI   Z
mask_ratiorf   NLr   r   r   r   Znoiser   r   r   r   r   r@   r@   rA   r     s$   $
r   rI   rM   returnc                 C   s   t j| d|jdS )Nr    rQ   r2   rX   r   rI   rM   r@   r@   rA   rn     s
   rn   c                 C   s   t j| d|jd dS )Nr    .r   rQ   r   r   r@   r@   rA   rp     s
   rp   r    	manhattanmax_positionsattention_headsdimsdistancec              	      sN   fdd | }|}t  |}|dkr+t t |dt |d d }nh|dkr|dkr8dd	 }n|d
kr@dd	 }t| }	|	 sMJ |	t|	}	t 	| | f}t
|	D ]6}
t
|	D ](}t
|	D ]!}t
|	D ]}|
|	 | }||	 | }||
||| |||f< qnqhqbq\td| |dd|d|dd }|S )Nc                    s\   dd }t |  r|| S dt t |  }|| d| dd d d | |   S )Nc                    s6   ddt | d       fddt| D S )N      c                    s   g | ]} |  qS r@   r@   )rb   iZratior   r@   rA   re         zPget_alibi.<locals>.get_slopes.<locals>.get_slopes_power_of_2.<locals>.<listcomp>)mathlog2ri   )nr@   r   rA   get_slopes_power_of_2  s   z<get_alibi.<locals>.get_slopes.<locals>.get_slopes_power_of_2r   r   )r   r   
is_integerfloor)r   r   Zclosest_power_of_2
get_slopesr@   rA   r     s   	 zget_alibi.<locals>.get_slopesr    r   rO   r   r   c                 S   s   t | | t ||  S rH   )absx1y1Zx2y2r@   r@   rA   <lambda>  r   zget_alibi.<locals>.<lambda>Z	euclideanc                 S   s   t | | d || d  S )Nr   )r   sqrtr   r@   r@   rA   r   	  s    z"unsupported number of alibi dims: )r2   r   r   ZarangerD   r   r   r   r`   r3   ri   	Exceptionrs   )r   r   r   r   ZmaxposZ
attn_headsZslopesZpos_biasZdfr   r   jrF   lZnew_xZnew_yr)   r@   r   rA   	get_alibi  s:   (

$r   c                 C   s   | d| d| }|  |d }	|| }
|	d u s0|	d|
k s0|	d|k s0|	j|ks0|	j|krdt||	d ur;|	dnd}t|
|	d urI|	dnd| }t||||dj||d|dd}	|	| |< |	d |
d |d |f }|||||}|S )Nrv   r   r    )r   r   r   )	getr6   r!   r(   maxr   rl   repeatrk   )Zalibi_biasesr$   r%   r&   r!   r(   r   r   	cache_keyZbufferedZtarget_sizeZbtZbnbr@   r@   rA   r   !  s0   
 
r   c                 C   s   |  d|ksJ | j| j|ksJ | j| j|ks J | j|  d|k r?t||  d d }tj| ||||fdd} | |ddd| } | dd |d |f S )Nr    rO   r   Z	replicate)mode.)	r6   rT   r!   r(   r   ceilrS   rt   rs   )r)   r$   r%   r&   r'   r!   r(   Zpszr@   r@   rA   r<   G  s   	r<   c                 C   sv   |  d}| }|jdd d}tj|d|d|d|j dd} tj| d|ddd||  ddd} | S )Nr    r   rO   rQ   )r6   r   rD   r2   rX   rs   r   Z	transpose)r)   rM   HZ	orig_biasrR   r@   r@   rA   rr   \  s   
rr   )r    r   ),loggingr   Znumpyr   r2   Ztorch.nnr0   Ztorch.nn.functionalZ
functionalrS   collectionsr   dataclassesr   	functoolsr   Z	omegaconfr   r   typingr   r   Z)funasr.models.emotion2vec.fairseq_modulesr	   r
   r   	getLoggerr   loggerr   r   r   r   r   r   r   rn   rp   r`   r   r   r   r<   rr   r@   r@   r@   rA   <module>   sR   
   	 
G
&