o
    4ѹg$                     @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
l m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z9 d dl:m;Z; ej<ddddefddZ=dd Z<e>dkre=  dS dS )    N)BytesIO)nullcontext)
DictConfig	OmegaConf)autocast
GradScaler)DistributedDataParallel)FullyShardedDataParallel)Join)ShardedGradScaler)SummaryWriter)average_checkpoints)tables)optim_classes)Trainer)scheduler_classes)
initialize)download_model)mark_only_lora_as_trainable)set_all_random_seed)load_pretrained_model)prepare_model_dir)model_summary)	AutoModel)Zconfig_nameZversion_basekwargsc                 C   st   |  ddrdd l}|  d| v sJ d| vr1td|  dd tdd	|  d	d
i| } tdi |  d S )NdebugFr   modelZ
model_confz"download models from model hub: {}ZhubmsZis_trainingT )getpdb	set_tracelogginginfoformatr   main)r   r    r   r   T/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/funasr/bin/train.py
main_hydra(   s   r'   c                   K   s  t | dd | dtjjjtjj_| dtjjjtjj_| ddtjj_| ddtjjj	_
ttjdd}|dkrDt  d	tjv oQttjd	 d
k}| dd}|s\|rmtj| dddd tj| td | dd}d| d< tdPi | }|s|rt dks|s|s|dkrtdPi |  |j} || d< | d }| d }|j}| d= | dd }|d urd|v rt|}t|ttfs|f}td| |D ]"}	| D ]\}
}|
|	d s|
|	krtd|
 d d|_ qq|dkrtt!|  |r||}t"||g| di ddd }n<|rPtd!fd"t#j$d#t%d$td%td&t%f
d'd(}t&j'|td)d*}tj| t(||d tj) d+}n
|j*| ddd,}t+|, j-| d< td- | d.d/}|t.v suJ t.|}||, fi | d0}td1 | d2d3}|t/v sJ t/|}||fi | d4}td5 tj0| d6 d7d8}|dPi | }t1dP|||| d | d9d:d;| d}|j2rt3|j2d<nd }|j4rt5|j2d<n|}|j6||||d= tj78| d9d>}tj9|dd? zt:|}W n   d }Y d@\}}t;|j<|j=D ]}t>? }t;|j@|jAD ]m}t>? }|jB|||jCdA\}}|jD||||||||||jA|jCdB d|_CtjE  t>? | dC }tdD| dE|dFdG|jA dH|jA|  dI|jA| | dFdJ|j=|  dK|j=| d
 |jA |jA | | dFdL q.d|_@|jF|||d
 |dM |G  d|_H|jI|d
 ||||d= t>? }|| dC }tdD| dE|dFdG|j= dN|j=| | dFdL	 dO|_JdO|_Kq!|jLdkrtM|jN|jO |P  d S )QNseedr   Zcudnn_enabledZcudnn_benchmarkZcudnn_deterministicTZenable_tf32Z
LOCAL_RANKZ
WORLD_SIZE   use_fsdpFbackendZncclzenv://)r+   Zinit_methodz Build model, frontend, tokenizerdevicecudacpu	tokenizerfrontendr   freeze_param,zfreeze_param is not None: %s.zSetting z.requires_grad = FalseZ
train_conffind_unused_parameters)Z
device_idsr4   g    חAmodulerecursenonwrapped_numelmin_num_paramsreturnc                 S   s*   t |k}tdd |  D dk}|o|S )Nc                 S   s   h | ]}|j qS r   )requires_grad).0pr   r   r&   	<setcomp>   s    z8main.<locals>.custom_auto_wrap_policy.<locals>.<setcomp>r)   )Zunwrapped_paramslen
parameters)r5   r6   r7   r8   Zis_largeZrequires_grad_uniformr   r   r&   custom_auto_wrap_policy~   s   z%main.<locals>.custom_auto_wrap_policyg     j@)r8   )Zauto_wrap_policyZmixed_precisionZ	device_id)r,   zBuild optimoptimZadamZ
optim_confzBuild scheduler	schedulerZwarmuplrZscheduler_confzBuild dataloaderZdataset_conf
dataloaderZDataloaderMapStyle
output_dirz./exp)
local_rankuse_ddpr*   r,   rD   )enabled)r   rA   rB   scalerZtensorboard)exist_ok)NN)data_split_i
start_step)r   rA   rB   rH   Zdataloader_traindataloader_valepochwriterrJ   data_split_numrK   g      @zrank: z, time_escaped_epoch: z.3fz hours, estimated to finish z data_slices, remaining: z	 slices, z hours, epoch: z	 epochs, z hours
)r   rL   rM   rN   z epoch: g        r   )Qr   r   torchbackendsZcudnnrG   Z	benchmarkZdeterministicr-   matmulZ
allow_tf32intosenvironr   printdistZinit_process_groupZ
set_devicer"   r#   r   Zget_rankr   r   r   eval
isinstancelisttupleZnamed_parameters
startswithr:   r   DDPnnModulebool	functoolspartialFSDPZcurrent_devicetonextr?   r,   r   r   Zdataloader_classesr   Zuse_fp16r   r*   r   Zresume_checkpointpathjoinmakedirsr   rangeZstart_epochZ	max_epochtimeperf_counterZstart_data_split_irO   Z
build_iterrK   Ztrain_epochZempty_cacheZvalidate_epochstepZstep_in_epochZsave_checkpointZtrain_acc_avgZtrain_loss_avgZrankr   rD   Zavg_nbest_modelclose) r   rE   rF   r*   r,   r   r/   r0   r1   tkr<   r@   Zmy_auto_wrap_policyrA   Zoptim_classrB   Zscheduler_classZdataloader_classrC   ZtrainerrH   Ztensorboard_dirrN   Zdataloader_trrL   rM   Ztime1rJ   Ztime_slice_iZtime_escapedZtime2r   r   r&   r%   7   st  










	

"
r%   __main__)?rT   sysrP   Ztorch.nnr^   Zhydrar"   rj   argparseior   
contextlibr   Ztorch.distributedZdistributedrW   Z	omegaconfr   r   Ztorch.cuda.ampr   r   Ztorch.nn.parallelr   r]   Ztorch.distributed.fsdpr	   rc   Z!torch.distributed.algorithms.joinr
   Z*torch.distributed.fsdp.sharded_grad_scalerr   ZtensorboardXr   Z'funasr.train_utils.average_nbest_modelsr   Zfunasr.registerr   Zfunasr.optimizersr   Zfunasr.train_utils.trainerr   Zfunasr.schedulersr   Zfunasr.train_utils.initializer   Z'funasr.download.download_model_from_hubr   Zfunasr.models.lora.utilsr   Z&funasr.train_utils.set_all_random_seedr   Z(funasr.train_utils.load_pretrained_modelr   Zfunasr.utils.miscr   Z funasr.train_utils.model_summaryr   Zfunasrr   r%   r'   __name__r   r   r   r&   <module>   sL    O
