o
    4ѹg#                     @   s   d dl Z d dlmZmZmZmZmZ d dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ ed
dG dd de
jZdS )    N)UnionDictListTupleOptional)CTC)force_gatherable)load_audio_text_image_videoextract_fbank)postprocess_utils)DatadirWriter)tables)
HypothesisZmodel_classesr   c                       s  e Zd ZdZ														d+d	ed
ededededededededededededef fddZde	j
de	j
de	j
de	j
dee	j
eee	j
f e	j
f f
ddZde	j
de	j
dee	j
e	j
f fd d!Zd"e	j
d#e	j
d$e	j
d%e	j
fd&d'Z				d,d(efd)d*Z  ZS )-Transformerz*CTC-attention hybrid Encoder-Decoder modelNP   r         Fspecaugspecaug_conf	normalizenormalize_confencoderencoder_confctc_conf
input_size
vocab_size	ignore_idblank_idsoseoslength_normalized_lossc                    s   t    |d urtj|}|di |}|d ur'tj|}|di |}tj|}|dd|i|}| }|d u r@i }td|	|d|}|| _	|d urS|n|	d | _
|d ur^|n|	d | _|	| _|
| _|| _|| _|| _d | _|| _|| _d S )Nr   )Zodimencoder_output_sizer    )super__init__r   Zspecaug_classesgetZnormalize_classesZencoder_classesZoutput_sizer   r   r   r    r   r   r   r   r   error_calculatorctcr!   )selfr   r   r   r   r   r   r   r   r   r   r   r   r    r!   kwargsZspecaug_classZnormalize_classZencoder_classr"   r(   	__class__r#   [/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/funasr/models/ctc/model.pyr%      s0   

zTransformer.__init__speechspeech_lengthstexttext_lengthsreturnc                 K   s   t | dkr|dddf }t | dkr |dddf }|jd }| ||\}}d\}	}
t }| ||||\}	}
|	}t| |d< | j	rTt
|d  }t|||f|j\}}}|||fS )zEncoder + Decoder + Calc loss
        Args:
                speech: (Batch, Length, ...)
                speech_lengths: (Batch, )
                text: (Batch, Length)
                text_lengths: (Batch,)
        r   Nr   )NNloss)lensizeshapeencodedict_calc_ctc_losstorchclonedetachr!   intsumr   device)r)   r.   r/   r0   r1   r*   
batch_sizeencoder_outencoder_out_lensloss_ctccer_ctcstatsr3   Zweightr#   r#   r-   forwardF   s"   

zTransformer.forwardc                 K   sR   | j dur| jr|  ||\}}| jdur| ||\}}| ||\}}||fS )zFrontend + Encoder. Note that this method is used by asr_inference.py
        Args:
                speech: (Batch, Length, ...)
                speech_lengths: (Batch, )
                ind: int
        N)r   trainingr   r   )r)   r.   r/   r*   rA   rB   r#   r#   r-   r7   s   s   
zTransformer.encoderA   rB   ys_padys_pad_lensc                 C   sR   |  ||||}d }| js%| jd ur%| j |j}| j| | dd}||fS )NT)Zis_ctc)r(   rG   r'   argmaxdatacpu)r)   rA   rB   rH   rI   rC   rD   Zys_hatr#   r#   r-   r9      s   zTransformer._calc_ctc_losskeyc           #         s&  | dddkrtdi }t|tjr=| dddkr=||}}	t|jdk r3|d d d d d f }|	d u r<|jd }	nKt }
t	||j
| dd	| dd|d
}t }||
 d|d< t|| dd|d\}}	t }|| d|d< |	  |j |j d |d< |j|d d}|	j|d d}	 ||	\}}t|tr|d } j|}g }| \}}}t|d ttfr|d }t||k r|| }t|D ]}||d || d d f }|jdd}tj|dd}tj jg|   jg |jd}t|dg}t |D ]\}}d }| dd ur1t! ds't"| d _# j#|d  d }d}t|j$trB|j$d| }n	|j$d|  }tt% fdd|}|&|}|'|}t()|\} }!|| || d}"|*|" |d urd+||d || < | |d || < qq||fS ) Nr@   r   z!batch decoding is not implemented	data_typeZsoundZfbank   fsi>  )rP   Zaudio_fsrN   	tokenizerz0.3fZ	load_data)rN   frontendZextract_feati  Zbatch_data_timer?   )r?   r   r   )Zdim)yseq
output_dirwriterZ
best_recogc                    s   |  j ko|  jko|  jkS )N)r    r   r   )xr)   r#   r-   <lambda>   s    z'Transformer.inference.<locals>.<lambda>)rM   tokenr0    rY   r0   ),r&   NotImplementedError
isinstancer:   Tensorr4   r6   timeperf_counterr	   rP   r
   r>   itemZframe_shiftZlfr_ntor7   tupler(   Zlog_softmaxr5   listrangerJ   Zunique_consecutiveZtensorr   tolistr    r?   r   	enumeratehasattrr   rU   rS   filterZ
ids2tokensZtokens2textr   Zsentence_postprocessappendjoin)#r)   Zdata_inZdata_lengthsrM   rQ   rR   r*   Z	meta_datar.   r/   Ztime1Zaudio_sample_listZtime2Ztime3rA   rB   Z
ctc_logitsresultsbndirV   rS   Z
nbest_hypsZ	nbest_idxZhypZibest_writerZlast_posZ	token_intrY   r0   Ztext_postprocessed_Zresult_ir#   rW   r-   	inference   s   







$



!zTransformer.inference)NNNNNNNr   r   r   r   r   r   F)NNNN)__name__
__module____qualname____doc__strr8   r=   boolr%   r:   r]   r   r   rF   r7   r9   rc   rq   __classcell__r#   r#   r+   r-   r      s    	
1
-

r   )loggingtypingr   r   r   r   r   r^   r:   Ztorch.nnnnZfunasr.models.ctc.ctcr   Zfunasr.train_utils.device_funcsr   Zfunasr.utils.load_utilsr	   r
   Zfunasr.utilsr   Zfunasr.utils.datadir_writerr   Zfunasr.registerr   Zfunasr.models.paraformer.searchr   registerModuler   r#   r#   r#   r-   <module>   s    
