o
    /ѹgU                     @   sF  d dl Z d dlZd dlmZmZmZ d dlZd dlZ	d dl
m  mZ d dlZd dlm  mZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lm Z  d d
l!m"Z" d dl#m$Z$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z,m-Z- dZ.e) Z/e"j0ej1dG dd de Z2G dd de	j3Z4dS )    N)DictOptionalUnion)autocast)Dataset)tqdm)Trainers)Model
TorchModel)	MsDataset)BaseTrainer)TRAINERS)DEFAULT_MODEL_REVISION	ModelFile)create_device)
get_logger)get_dist_infoget_local_rank	init_distsi-snr)module_namec                   @   s   e Zd ZdZdddefdededee deeee	f  deeee	f  dee fd	d
Z
dejjfddZdd Zdedeeef fddZdS )SeparationTraineraa  A trainer is used for speech separation.

    Args:
        model: id or local path of the model
        work_dir: local path to store all training outputs
        cfg_file: config file of the model
        train_dataset: dataset for training
        eval_dataset: dataset for evaluation
        model_revision: the git version of model on modelhub
    Nmodelwork_dircfg_filetrain_dataseteval_datasetmodel_revisionc              	   K   sj  t |tr| ||| _|d u rtj| jtj}n|d us"J dtj	|| _t
| | |  | _|| _|dd d urEt|d  t \}}	|	dk| _|dd}
| jr`t }d| }
t|
| _d|vr{t| jjdstJ d| jjj| _n|d | _|| _|| _tj| jd	}| j| jjj| jjjj| jjjj | jjjj!| jjj"j#| jjj"j$| jjj"j%d
}ddl&m'} t(|}|||d| _)W d    n1 sw   Y  t*j+| j||d ddddddd}| jj,dkr| jj, d| jj- |d< t*j.j/0| j| _1| j1| j)d< | j)d 2d| j1i | j3 }| j)d 2| t4|| j)d | j)|| j)d d| _5d S )Nz?Config file should not be None if model is not from pretrained!launcher   deviceZgpuzcuda:
max_epochsz1max_epochs is missing from the configuration filezhparams.yaml)output_folderseedlrweight_decayclip_grad_normfactorpatiencedont_halve_until_epochr   )load_hyperpyyaml)	overrides)Zexperiment_directoryZhyperparams_to_saver+   FcpuZnccl)debugr    Zdata_parallel_backendZdistributed_launchZdistributed_backendZfind_unused_parametersZcuda:epoch_countercheckpointercounter	optimizer)modulesZ	opt_classhparamsrun_optsr0   )6
isinstancestrZget_or_download_model_dir	model_dirospathjoinr   ZCONFIGURATIONdirnamer   __init__build_modelr   r   getr   r   _distr   r   r    hasattrcfgtrainr!   Z_max_epochsr   r   r#   r2   r$   r%   r&   lr_schedulerr'   r(   r)   Zhyperpyyamlr*   openr4   sbZcreate_experiment_directorytypeindexutilsZ
epoch_loopZEpochCounterr/   Zadd_recoverablesZas_dict
Separation	separator)selfr   r   r   r   r   r   kwargs_Z
world_sizeZdevice_nameZ
local_rankZhparams_filer+   r*   Zfinr5   r3    rO   p/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/modelscope/trainers/audio/separation_trainer.pyr=   .   s   
	















zSeparationTrainer.__init__returnc                 C   sD   t j| j| jdd}t|trt|dr|jS t|tj	j
r |S dS )z1 Instantiate a pytorch model and return.
        T)Zcfg_dictZtrainingr   N)r	   Zfrom_pretrainedr8   rB   r6   r
   rA   r   torchnnModule)rL   r   rO   rO   rP   r>      s   
zSeparationTrainer.build_modelc                 O   s,   | j j| j| j| j| jd | jd d d S )Ndataloader_opts)Ztrain_loader_kwargsZvalid_loader_kwargs)rK   Zfitr/   r   r   r4   )rL   argsrM   rO   rO   rP   rC      s   
zSeparationTrainer.traincheckpoint_pathc                 O   sB   |r|| j j_n| jj| jd | jj| j| j d t	d}t	|iS )N)r    rU   )Ztest_loader_kwargsZmin_key)
r4   r0   Zcheckpoints_dirr   Zload_check_pointr    rK   evaluater   EVAL_KEY)rL   rW   rV   rM   valuerO   rO   rP   rX      s   zSeparationTrainer.evaluate)__name__
__module____qualname____doc__r   r7   r   r   r   r   r=   rR   rS   rT   r>   rC   r   floatrX   rO   rO   rO   rP   r   !   s0    
c
	
r   c                   @   sb   e Zd ZdZdddZdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd ZdS )rJ   z:A subclass of speechbrain.Brain implements training steps.Nc                    s  |\}}|  j|  j}}tjfddt jjD dd  j|tjj	krmt
 4  jjs: jjrG |\}d} jjrR j||} jjr^ |\}W d   n1 shw   Y   jd |} jd |}t|g jj }|| tj fddt jjD dd}|d	}	|d	}
|	|
krt|d
d
d
|	|
 f}|fS |ddd|	ddf }|fS )z?Forward computations from the mixture to the separated signals.c                    s   g | ]} | d   dqS )r   )	unsqueeze.0i)targetsrO   rP   
<listcomp>   s    z.Separation.compute_forward.<locals>.<listcomp>r`   ZdimNencoderZmasknetc                    s$   g | ]} j d  | dqS )decoderr`   )r3   ra   rb   )rL   sep_hrO   rP   rf      s    r   r   )tor    rR   catranger4   num_spksrF   StageTRAINno_graduse_speedperturbuse_rand_shiftadd_speed_perturbsumZuse_wavedropZwavedropZlimit_training_signal_lencut_signalsr3   stacksizeFpad)rL   mixre   stageZnoiseZmix_lensZmix_wZest_maskZ
est_sourceZT_originZT_estrO   )rL   rj   re   rP   compute_forward   sL   

	




zSeparation.compute_forwardc                 C   s   | j ||S )zComputes the sinr lossN)r4   loss)rL   predictionsre   rO   rO   rP   compute_objectives   s   zSeparation.compute_objectivesc                 C   sf  |j }|j|jg}| jjdkr||j | jrt ; | 	||t
jj\}}| ||}| jjrK| jj}|||k }| dkrF| }n	td n| }W d   n1 sYw   Y  || jjk r| dkr| j|  | jjdkr| j| j tjj| j | jj | j | j | j!  n|  j"d7  _"t#$d%| j" t&d'| j(|_)nt| 	||t
jj\}}| ||}| jjr| jj}|||k }| dkr| }n| }|| jjk r| dkr|  | jjdkrtjj| j | jj | j   n|  j"d7  _"t#$d%| j" t&d'| j(|_)| j*  |+ , S )zTrains one batch   r   zloss has zero elements!!Nr   zNinfinite loss or empty loss! it happened {} times so far - skipping this batch)-mix_sigs1_sigs2_sigr4   rn   appends3_sigZauto_mix_precr   r}   rF   ro   rp   r   Zthreshold_byloss	thresholdZnelementmeanprintZloss_upper_limZscalerZscaleZbackwardr&   Zunscale_r2   rR   rS   rI   Zclip_grad_norm_r3   
parametersstepupdateZnonfinite_countloggerinfoformatZtensorrk   r    dataZ	zero_graddetachr,   )rL   batchmixturere   r   r~   thZloss_to_keeprO   rO   rP   	fit_batch   sz   




zSeparation.fit_batchc                 C   s   |j }|j}|j|jg}| jjdkr||j t	  | 
|||\}}| ||}W d   n1 s6w   Y  |tjjkrn| jjrnt| jdrd| jjdkrc| |d ||| | j jd7  _n
| |d ||| |  S )z/Computations needed for validation/test batchesr   Nn_audio_to_saver   r`   )idr   r   r   r4   rn   r   r   rR   rq   r}   r   rF   ro   TEST
save_audiorA   r   r   r   )rL   r   r|   snt_idr   re   r   r~   rO   rO   rP   evaluate_batch5  s&   
zSeparation.evaluate_batchc                 C   s   d|i}|t jjkr|| _|t jjkrVt| jjtj	r/| j| j
g||\}}t| j
| n
| jj
jjd d }| jjj||d| j|d | jjd|d idgd dS dS )z"Gets called at the end of a epoch.r   r   r$   )epochr$   )Z
stats_metatrain_statsZvalid_stats)metaZmin_keysN)rF   ro   rp   r   ZVALIDr6   r4   rD   
schedulersZReduceLROnPlateaur2   Zupdate_learning_rateZoptimZparam_groupsZtrain_loggerZ	log_statsr0   Zsave_and_keep_only)rL   r|   Z
stage_lossr   Zstage_statsZ
current_lrZnext_lrrO   rO   rP   on_stage_endM  s0   


zSeparation.on_stage_endc           
      C   sh  d}d}| j jrg }d}t|jd D ]-}| j |dddd|f |}|| |dkr4|jd }q|jd |k r@|jd }q| j jrud}t|jd D ]&}t| j j	| j j
d}|| | j||< tj|| |d fdd||< qN|r| j jrtj|jd ||jd |jtjd	}t|D ]\}}|| ddd|f |dddd|f< q|d}	|	|fS )
z=Adds speed perturbation and random_shift to the input signalsr`   FTNr   r   r   )Zshiftsdims)r    Zdtype)r4   rr   rm   shapeZspeedperturbr   rs   rR   randintZ	min_shiftZ	max_shiftrk   r    Zrollzerosr_   	enumerateru   )
rL   re   Z	targ_lensZmin_lenZ	recombineZnew_targetsrd   
new_targetZ
rand_shiftr{   rO   rO   rP   rt   m  sL   


,
zSeparation.add_speed_perturbc                 C   sp   t ddtd|jd | jj  d }|dd||| jj ddf }|dd||| jj f }||fS )zThis function selects a random segment of a given length within the mixture.
        The corresponding targets are selected accordinglyr   r   r   N)rR   r   maxr   r4   Ztraining_signal_lenitem)rL   r   re   Z	randstartrO   rO   rP   rv     s    
zSeparation.cut_signalsc                 C   s6   t |dr	|  | D ]}||kr| | qdS )z3Reinitializes the parameters of the neural networksreset_parametersN)rA   r   r3   reset_layer_recursively)rL   ZlayerZchild_layerrO   rO   rP   r     s   

z"Separation.reset_layer_recursivelyc                 C   s*  ddl m} tj| jjd}g }g }g }g }g d}tjj	j
|fi | jj}	t|d(}
tj|
|d}|  t|	dd}t|D ]\}}|j\}}|j}|j|jg}| jjd	krd||j t  | |j|tjj\}}W d
   n1 sw   Y  | ||}tj|g| jj dd}| |j!}| ||}|" |"  }||d # $ % |d # & $ % \}}}}||d # $ % |d # & $ % \}}}}|" |"  }|d |" ||'  |'  d}|(| ||"  ||"  ||'   ||'   qFdt)*|" t)*|" t)*|" t)*|" d}|(| W d
   n	1 sJw   Y  W d
   n	1 sZw   Y  t+,d-t)*|"  t+,d-t)*|"  t+,d-t)*|"  t+,d-t)*|"  d
S )zVThis script computes the SDR and SI-SNR metrics and saves
        them into a csv filer   )bss_eval_sourcesztest_results.csv)r   sdrsdr_ir   zsi-snr_iw)
fieldnamesT)Zdynamic_ncolsr   Nr`   rg   avgzMean SISNR is {}zMean SISNRi is {}zMean SDR is {}zMean SDRi is {}).Zmir_eval.separationr   r9   r:   r;   r4   r"   rF   ZdataioZ
dataloaderZmake_dataloaderrU   rE   csv
DictWriterwriteheaderr   r   r   r   r   r   rn   r   r   rR   rq   r}   ro   r   r   rw   rk   r    r   tr,   numpyr   r   writerownparrayr   r   r   )rL   Z	test_datar   	save_fileZall_sdrsZ
all_sdrs_iZ
all_sisnrsZall_sisnrs_iZcsv_columnsZtest_loaderZresults_csvwriterr   rd   r   r   Zmix_lenr   re   r   ZsisnrZmixture_signalZsisnr_baselineZsisnr_ir   rN   Zsdr_baseliner   rowrO   rO   rP   save_results  s   



DzSeparation.save_resultsc           	   	   C   sN  t j| jjd}t j|st | t| jjD ]^}|ddd|f }||	 
  d }t j|d||d }t||d | jj |ddd|f }||	 
  d }t j|d||d }t||d | jj q|d dddf }||	 
  d }t j|d|}t||d | jj dS )	zFsaves the test audio (mixture, targets, and estimated sources) on diskZaudio_resultsr   Ng      ?zitem{}_source{}hat.wavr   zitem{}_source{}.wavzitem{}_mix.wav)r9   r:   r;   r4   Zsave_folderexistsmkdirrm   rn   absr   r   
torchaudiosavera   r,   Zsample_rate)	rL   r   r   re   r   Z	save_pathnssignalr   rO   rO   rP   r     s8   
zSeparation.save_audio)N)r[   r\   r]   r^   r}   r   r   r   r   rt   rv   r   r   r   rO   rO   rO   rP   rJ      s    
6F /^rJ   )5r   r9   typingr   r   r   r   r   ZspeechbrainrF   Zspeechbrain.nnet.schedulersZnnetr   rR   Ztorch.nn.functionalrS   Z
functionalry   r   Ztorch.cuda.ampr   Ztorch.utils.datar   r   Zmodelscope.metainfor   Zmodelscope.modelsr	   r
   Zmodelscope.msdatasetsr   Zmodelscope.trainers.baser   Zmodelscope.trainers.builderr   Zmodelscope.utils.constantr   r   Zmodelscope.utils.devicer   Zmodelscope.utils.loggerr   Zmodelscope.utils.torch_utilsr   r   r   rY   r   Zregister_moduleZspeech_separationr   ZBrainrJ   rO   rO   rO   rP   <module>   s6    