o
    4ѹg                     @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
l m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z9 zd dl:Z:W n   dZ:Y ej;ddddefddZ<dd Z;e=dkre<  dS dS )    N)BytesIO)nullcontext)
DictConfig	OmegaConf)autocast
GradScaler)DistributedDataParallel)FullyShardedDataParallel)Join)ShardedGradScaler)average_checkpoints)tables)optim_classes)Trainer)scheduler_classes)
initialize)download_model)mark_only_lora_as_trainable)set_all_random_seed)load_pretrained_model)prepare_model_dir)model_summary)	AutoModel)Zconfig_nameZversion_basekwargsc                 C   st   |  ddrdd l}|  d| v sJ d| vr1td|  dd tdd	|  d	d
i| } tdi |  d S )NdebugFr   modelZ
model_confz"download models from model hub: {}ZhubmsZis_trainingT )getpdb	set_tracelogginginfoformatr   main)r   r   r   r   W/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/funasr/bin/train_ds.py
main_hydra,   s   r&   c                  K   s$  t | dd | dtjjjtjj_| dtjjjtjj_| ddtjj_| ddtjjj	_
ttjdd}ttjd	d}ttjd
d}|dkrVt  |dk}| dd}| dd}|r{td|  tj| ddd n |s|rtd| d|  tj| dddd tj| td | dd}d| d< td@i | }|dkrtd@i |  |j} || d< | d }	| d }
|j}| d= | dd }|d urd|v rt|}t|ttfs|f}td | |D ]$}| D ]\}}| |d! s||krtd"| d# d|_!qq|dkr(tt"|  t#d@|||||| d | d$d | d%d&d'| d(}|j$|fi | }ttjd	d| d< ttjd	d|_%|j&|fi | \}}}td) tj'| d* d+d,}|d@i | }|j(s|j)rt*dd-nd }|j+rt,|j(d-n|}|j-||||d. d/\}}t.|j/|j0D ]}t12 }t.|j3|j4D ]l}t12 }|j5|||j6d0\}}|j7|||||||||j4|j6d1
 d|_6tj8  t12 | d2 }td3| d4|d5d6|j4 d7|j4|  d8|j4| | d5d9|j0|  d:|j0| d |j4 |j4 | | d5d; qd|_3|j9|||d d< |:  d|_;|j<|d ||||d. t12 }|| d2 }td3| d4|d5d6|j0 d=|j0| | d5d;	 d>|_=d>|_>q|j?dkrt@|jA|jB|jCd? |D  d S )ANseedr   Zcudnn_enabledZcudnn_benchmarkZcudnn_deterministicTZenable_tf32ZRANKZ
LOCAL_RANKZ
WORLD_SIZE   use_fsdpFuse_deepspeedzuse_deepspeed: backendZnccl)Zdist_backendz	use_ddp: z, use_fsdp: zenv://)r+   Zinit_methodz Build model, frontend, tokenizerdevicecudacpu	tokenizerfrontendr   freeze_param,zfreeze_param is not None: %s.zSetting z.requires_grad = Falseexcludes
output_dirz./exp)rank
local_rank
world_sizeuse_ddpr)   r,   r4   r5   Z
train_confzBuild dataloaderZdataset_conf
dataloaderZDataloaderMapStyle)enabled)r   optim	schedulerscaler)NN)data_split_i
start_step)
r   r<   r=   r>   Zdataloader_traindataloader_valepochr?   data_split_numr@   g      @z

rank: z, time_escaped_epoch: z.3fz hours, estimated to finish z data_slices, remaining: z	 slices, z hours, epoch: z	 epochs, z hours
)r   rA   rB   z epoch: g        )r*   r   )Er   r   torchbackendsZcudnnr;   Z	benchmarkZdeterministicr-   matmulZ
allow_tf32intosenvironr   printr!   r"   	deepspeedZinit_distributeddistZinit_process_groupZ
set_devicer   r   r   r   eval
isinstancelisttupleZnamed_parameters
startswithZrequires_gradr   r   Z
warp_modelr,   Zwarp_optim_schedulerZdataloader_classesZuse_fp16Zuse_bf16r   r)   r   Zresume_checkpointrangeZstart_epochZ	max_epochtimeperf_counterZstart_data_split_irC   Z
build_iterr@   Ztrain_epochZempty_cacheZvalidate_epochstepZstep_in_epochZsave_checkpointZtrain_acc_avgZtrain_loss_avgr6   r   r5   Zavg_nbest_modelr*   close)r   r6   r7   r8   r9   r)   r*   r,   r   r/   r0   r1   tkpZtrainerr<   r=   Zdataloader_classr:   r>   Zdataloader_trrA   rB   Ztime1r?   Ztime_slice_iZtime_escapedZtime2r   r   r%   r$   ;   s  





	


"
r$   __main__)>rH   sysrD   Ztorch.nnnnZhydrar!   rS   argparseior   
contextlibr   Ztorch.distributedZdistributedrL   Z	omegaconfr   r   Ztorch.cuda.ampr   r   Ztorch.nn.parallelr   ZDDPZtorch.distributed.fsdpr	   ZFSDPZ!torch.distributed.algorithms.joinr
   Z*torch.distributed.fsdp.sharded_grad_scalerr   Z'funasr.train_utils.average_nbest_modelsr   Zfunasr.registerr   Zfunasr.optimizersr   Zfunasr.train_utils.trainer_dsr   Zfunasr.schedulersr   Zfunasr.train_utils.initializer   Z'funasr.download.download_model_from_hubr   Zfunasr.models.lora.utilsr   Z&funasr.train_utils.set_all_random_seedr   Z(funasr.train_utils.load_pretrained_modelr   Zfunasr.utils.miscr   Z funasr.train_utils.model_summaryr   Zfunasrr   rK   r$   r&   __name__r   r   r   r%   <module>   sR    &
