o
    4ѹg.O                     @   s   d dl mZ d dlZd dlZd dlZd dlmZ d dlm	  m
Z
 d dlmZ d dlm  mZ d dlmZ dd Zdd Zd	d
 ZeddeddG dd dejZeddG dd dejZG dd dejZdS )    )TupleN)pad_sequence)tablesc                 C   s0  t | ddd}| }W d    n1 sw   Y  g }g }tt|D ]O}||  }|d dkrQ||d   }|d dkrP|dt|d  }t|}q%q%|d d	krt||d   }|d dkrt|dt|d  }t|}q%q%t|tj	}	t|tj	}
t|	|
g}t
j|t
j	d
}|S )Nrzutf-8)encodingr   z
<AddShift>   z<LearnRateCoef>   z	<Rescale>dtype)open	readlinesrangelensplitlistnparrayZastypefloat32torch	as_tensor)	cmvn_fileflinesZ
means_listZ	vars_listiZ	line_itemZadd_shift_lineZrescale_linemeansvarscmvn r   a/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/funasr/frontends/wav_frontend.py	load_cmvn   s4   
r   c                 C   sf   | j }| j}| j\}}|ddd|f }|ddd|f }| ||7 } | ||9 } | tjS )z"
    Apply CMVN with mvn data
    r   r   N   )devicer
   shapetotyper   r   inputsr   r!   r
   framedimr   r   r   r   r   
apply_cmvn)   s   
r)   c                 C   s
  g }| j d }tt|| }| d |d d d}t|| f} ||d d  }| j d }|| df}||| f}	|| | d }
|||
|   }|dkrwd| d|  |d |
 |  d ||
  }t| g| dd  gt|  } | |	|}| 	tj
S )Nr   r   r    )r"   intr   ceilrepeatr   vstack
as_stridedcloner$   r   )r&   lfr_mlfr_n
LFR_inputsTT_lfrZleft_paddingfeat_dimstridessizeslast_idxnum_paddingLFR_outputsr   r   r   	apply_lfr:   s    

,"r<   Zfrontend_classesZwav_frontendWavFrontendc                       s   e Zd ZdZ													
		d&dededededededededededededef fddZdefddZ	de
jdee
je
jf fdd Zde
jd!e
jdee
je
jf fd"d#Zde
jd!e
jdee
je
jf fd$d%Z  ZS )'r=   (Conventional frontend structure for ASR.N>  hammingP      
   r*   r         ?Tr   fswindown_melsframe_lengthframe_shiftfilter_length_minfilter_length_maxr1   r2   dither
snip_edgesupsacle_samplesc                    s|   t    || _|| _|| _|| _|| _|| _|| _|	| _	|
| _
|| _|| _|| _|| _| jd u r6d | _d S t| j| _d S N)super__init__rE   rF   rG   rH   rI   rJ   rK   r1   r2   r   rL   rM   rN   r   r   selfr   rE   rF   rG   rH   rI   rJ   rK   r1   r2   rL   rM   rN   kwargs	__class__r   r   rQ   R   s   
$zWavFrontend.__init__returnc                 C      | j | j S rO   rG   r1   rS   r   r   r   output_sizes      zWavFrontend.output_sizeinputc                 K   s,  | d}g }g }t|D ]c}|| }|| d | }	| jr"|	d }	|	d}	tj|	| jt| j|| j	 d | j
| jd| j| j	| jd	}
| jdksN| jdkrVt|
| j| j}
| jd urat|
| j}
|
 d}||
 || qt|}|dkr|d d d d d d f }||fS t|ddd}||fS )	Nr                )num_mel_binsrH   rI   rL   energy_floorwindow_typesample_frequencyrM   r   TZbatch_firstZpadding_value)sizer   rN   	unsqueezekaldifbankrG   minrH   rE   rI   rL   rF   rM   r1   r2   r<   r   r)   appendr   r   r   )rS   r]   input_lengthsrT   
batch_sizefeats
feats_lensr   waveform_lengthwaveformmatfeat_length	feats_padr   r   r   forwardv   sB   





zWavFrontend.forwardrl   c                 C   s   | d}g }g }t|D ]9}|| }|| d | }|d }|d}tj|| j| j| j| jd| j	| j
d}	|	 d}
||	 ||
 qt|}t|ddd}||fS )Nr   r^   r`   ra   rH   rI   rL   rb   rc   rd   Tre   )rf   r   rg   rh   ri   rG   rH   rI   rL   rF   rE   rk   r   r   r   rS   r]   rl   rm   rn   ro   r   rp   rq   rr   rs   rt   r   r   r   forward_fbank   s0   




zWavFrontend.forward_fbankc           
      C   s   | d}g }g }t|D ];}||d || d d f }| jdks&| jdkr.t|| j| j}| jd ur9t|| j}| d}|| || qt	|}t
|ddd}	|	|fS )Nr   r   Tr`   re   )rf   r   r1   r2   r<   r   r)   rk   r   r   r   )
rS   r]   rl   rm   rn   ro   r   rr   rs   rt   r   r   r   forward_lfr_cmvn   s   




zWavFrontend.forward_lfr_cmvnNr?   r@   rA   rB   rC   r*   r*   r   r   rD   TT)__name__
__module____qualname____doc__strr+   floatboolrQ   r[   r   Tensorr   ru   rx   ry   __classcell__r   r   rU   r   r=   M   s|    	
!
*
WavFrontendOnlinec                       sr  e Zd ZdZ													
		d6dededededededededededededef fddZdefddZ	e
dejdejdejfd d!Ze
	"d7dejdeded#edeejejef f
d$d%Ze
d&ed'ed(edefd)d*Zi fd+ejd,ejd-edeejejejf fd.d/Zd"i fd+ejd,ejd#ed-efd0d1Zd+ejd,ejfd2d3Zi fd-efd4d5Z  ZS )8r   z6Conventional frontend structure for streaming ASR/VAD.Nr?   r@   rA   rB   rC   r*   r   rD   Tr   rE   rF   rG   rH   rI   rJ   rK   r1   r2   rL   rM   rN   c                    s   t    || _|| _|| _|| _|| _t| j| j d | _t| j| j d | _	|| _
|| _|	| _|
| _|| _|| _|| _|| _| jd u rLd | _d S t| j| _d S )Nr_   )rP   rQ   rE   rF   rG   rH   rI   r+   frame_sample_lengthframe_shift_sample_lengthrJ   rK   r1   r2   r   rL   rM   rN   r   r   rR   rU   r   r   rQ      s"   
$zWavFrontendOnline.__init__rW   c                 C   rX   rO   rY   rZ   r   r   r   r[      r\   zWavFrontendOnline.output_sizer&   r   c                 C   s   | j }| j}| j\}}t|ddd|f |df}t|ddd|f |df}| t|||7 } | t|||9 } | tj	S )z*
        Apply CMVN with mvn data
        r   r   Nr    )
r!   r
   r"   r   Ztiler   
from_numpyr$   r#   r   r%   r   r   r   r)     s   
  zWavFrontendOnline.apply_cmvnFis_finalc                 C   s:  g }| j d }tt||d d  | }|}| j d }| }	|| df}
||| f}|| | d }||||   }|ri|dkrhd| d|  |d | |  d ||  }t| g| dd gt|  } n|dkru||| f}|}t|d || }| d| ||
}|	|dddf }| 	tj
||fS )z%
        Apply lfr with data
        r   r   r    r*   N)r"   r+   r   r,   r   r.   rj   r/   r0   r$   r   )r&   r1   r2   r   r3   r4   r5   Z
splice_idxr6   Z
ori_inputsr7   r8   r9   r:   r;   lfr_splice_cacher   r   r   r<     s0   

,"zWavFrontendOnline.apply_lfrsample_lengthr   r   c                 C   s,   t | | | d }|dkr| |kr|S dS )Nr   r   )r+   )r   r   r   	frame_numr   r   r   compute_frame_num6  s   z#WavFrontendOnline.compute_frame_numr]   rl   cachec                 K   s`  | d}tj|d |fdd}| |jd | j| j}|d d |jd || j   d f |d< td}td}td}	|rg }g }
g }	t|D ]B}|| }|	|d |d | j | j   |d }|
d}tj|| j| j| j| jd| j| jd}| d}|
	| |		| qLt|}t|	}	t|
d	dd
}||d< t|	|d< |||	fS )Nr   input_cacher   r(   r*   r^   r`   rv   Tre   fbanksfbanks_lens)rf   r   catr   r"   r   r   emptyr   rk   rg   rh   ri   rG   rH   rI   rL   rF   rE   stackr   r   copydeepcopy)rS   r]   rl   r   rT   rm   r   	waveformsrt   ro   rn   r   rq   rr   rs   r   r   r   rx   =  s^   
 








zWavFrontendOnline.forward_fbankc                 K   s   | d}g }g }g }	t|D ]J}
||
d ||
 d d f }| jdks(| jdkr9| || j| j|\}|d |
< }| jd urE| || j}| d}|| || |	| qt	
|}t|ddd}t	
|	}	|||	fS )Nr   r   r   Tr`   re   )rf   r   r1   r2   r<   r   r)   r   rk   r   r   r   )rS   r]   rl   r   r   rT   rm   rn   ro   lfr_splice_frame_idxsr   rr   Zlfr_splice_frame_idxrs   rt   r   r   r   ry   x  s(   







z"WavFrontendOnline.forward_lfr_cmvnc                 K   s  | dd}| di }t|dkr| | |jd }|dks$J d| j|||d\}}}	|jd r<tj|d |fdd	|d
< |d sit|D ]}
|d ||
 dd d f j	dd	
| jd d d qJ|	d |d d jd  | jkrt|d }tj||fdd	}|	|d jd 7 }	t|d
 jd | j | j d }|d  dkr| jd d nd}| j||	||d\}}	}| jdkrtd|d< ||	fS |d | }|d
 d d || j || j f |d< |d | j | j }|d
 d d d |f |d
< ||	fS |d
 d d d | j| j  f |d< t|D ]}
tj|d |
 ||
 fdd	|d |
< qtd|	fS |rp|d  dkrJ|n|d |d
< t|d }tj|tjd|jd  }	| j||	||d\}}	}||	fS )Nr   Fr   r   r   zOwe support to extract feature online only when the batch size is equal to 1 now)r   reserve_waveformsr   r   r   r    r	   )getr   
init_cacher"   rx   r   r   r   rk   rg   r-   r1   r   r+   r   r   Znumelry   r   zeros)rS   r]   rl   rT   r   r   rm   r   rn   Zfeats_lengthsr   Zlfr_splice_cache_tensorZframe_from_waveformsZminus_framer   Zreserve_frame_idxr   _r   r   r   ru     s   

," 
'zWavFrontendOnline.forwardc                 C   s@   t d|d< t d|d< g |d< d |d< d |d< d |d< |S )Nr   r   r   r   r   r   r   )r   r   )rS   r   r   r   r   r     s   zWavFrontendOnline.init_cacherz   )F)r{   r|   r}   r~   r   r+   r   r   rQ   r[   staticmethodr   r   r)   r   r<   r   dictrx   ry   ru   r   r   r   r   rU   r   r      s    	
)!

?
 Rc                       sv   e Zd ZdZ					ddededed	ed
ef
 fddZdefddZdejdejde	ejejf fddZ
  ZS )WavFrontendMel23r>   r?   rB   rC   r   rE   rH   rI   r1   r2   c                    s2   t    || _|| _|| _|| _|| _d| _d S )N   )rP   rQ   rE   rH   rI   r1   r2   rG   )rS   rE   rH   rI   r1   r2   rT   rU   r   r   rQ     s   
	
zWavFrontendMel23.__init__rW   c                 C   s   | j d| j d  S )Nr    r   rY   rZ   r   r   r   r[     s   zWavFrontendMel23.output_sizer]   rl   c                 C   s   | d}g }g }t|D ]D}|| }|| d | }| }t|| j| j}	t|	}	tj|	| j	d}	|	d d | j
 }	t|	}	|	 d}
||	 ||
 qt|}t|ddd}||fS )Nr   )Zcontext_sizeTr`   re   )rf   r   numpyeend_ola_featureZstftrH   rI   Z	transformZsplicer1   r2   r   r   rk   r   r   rw   r   r   r   ru     s$   





zWavFrontendMel23.forward)r?   rB   rC   r   r   )r{   r|   r}   r~   r+   rQ   r[   r   r   r   ru   r   r   r   rU   r   r     s4    r   )typingr   r   r   r   r   Ztorch.nnnnZtorchaudio.compliance.kaldiZ
compliancerh   Ztorch.nn.utils.rnnr   Z!funasr.frontends.eend_ola_featureZ	frontendsr   Zfunasr.registerr   r   r)   r<   registerModuler=   r   r   r   r   r   r   <module>   s*   

 
  "