o
    /ѹg                     @   s   d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZ d dlmZ e Zd	Zd
ZdZdZdZdZG dd dZdS )    )print_functionN)hash_url_to_filename)HubApi)DataDownloadConfig)MS_CACHE_HOME)"DEFAULT_DATA_ACCELERATION_ENDPOINTMetaDataFields
UploadMode)
get_loggerZAccessIdZAccessSecretZSecurityTokenBucketZ	BackupDirZDirc                
   @   s`   e Zd Zdd Zdd Zdd Zedd Zd	ed
e	fddZ
dededededef
ddZdS )OssUtilitiesc                 C   s~   | j |d || _|| _|| _tjtd| _t	
 | _d| _d| _d| _dd l}|j| jd| _|j| jd| _t | _d S )N)
oss_configztmp/resumable_storei   i      r   )root)_do_initdataset_name	namespacerevisionospathjoinr   Zresumable_store_root_pathmultiprocessing	cpu_countnum_threads	part_sizemultipart_thresholdmax_retriesoss2ZResumableDownloadStoreresumable_store_downloadZResumableStoreresumable_store_uploadr   api)selfr   r   r   r   r    r"   i/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/modelscope/msdatasets/utils/oss_utils.py__init__   s&   
zOssUtilities.__init__c                 C   s   dd l }|t | _|t | _|t | _tddkrt	| _
n	d|d  d| _
|t | _|| j| j| j}|j|| j
| jdd| _|t | _|t | _d S )	Nr   ZENABLE_DATASET_ACCELERATIONTruezhttps://ZRegionz.aliyuncs.comx   )connect_timeout)r   	ACCESS_IDkeyACCESS_SECRETsecretSECURITY_TOKENtokenr   getenvr   ZendpointBUCKETZbucket_nameZStsAuthr   bucketDIRoss_dirBACK_DIRoss_backup_dir)r!   r   r   authr"   r"   r#   r   1   s   




zOssUtilities._do_initc                 C   s2   t d | jj| j| jd| jd}| | d S )Nz"Reloading sts token automatically.T)r   r   Zcheck_cookier   )loggerinfor    Z!get_dataset_access_config_sessionr   r   r   r   )r!   Zoss_config_refreshr"   r"   r#   _reload_stsB   s   
zOssUtilities._reload_stsc                 C   s8   |rt dt| t|  }td|ddd d S d S )Nd   z{0}%  T)endflush)intfloatprintformat)Zconsumed_bytestotal_bytesZrater"   r"   r#   _percentageK   s   zOssUtilities._percentageoss_file_namedownload_configc              
   C   s:  dd l }|j}tj| j|}tj| j|}|j}d}|r,|j	|}	|	r,|		t
j}d}
	 z?|
d7 }
|r9|}n
| j|rA|n|}t|d d}tj||}|jsYtj|sl|j| j||| j| j| j| j| jd W |S  ty } z!|j	ddkr|   |
| jkrtd	|  |W Y d }~nd }~ww q/)
Nr   FT   )etag)storeZmultiget_thresholdr   progress_callbackr   status  zFailed to download )r   	cache_dirr   r   r   r2   r4   splitZmeta_args_mapgetr   ZARGS_BIG_DATAr0   object_existsr   Zforce_downloadexistsZresumable_downloadr   r   r   rB   r   	Exception__dict__r8   r   r6   warning)r!   rC   rD   r   rK   Zcandidate_keyZcandidate_key_backuprL   Zbig_dataZ	args_dictretry_countZfile_oss_keyfilenameZ
local_pather"   r"   r#   downloadQ   s`   	
zOssUtilities.downloadoss_object_namelocal_file_pathindicate_individual_progressupload_modereturnc              
   C   s   dd l }d}tj| j|}|r| j}nd }	 z6|d7 }| j|}	|tj	kr9|	r9t
d| d|j d W |S |j| j||| j| j| j|| jd W |S  tyq }
 z|
jdd	kra|   || jkrg W Y d }
~
nd }
~
ww q)
Nr   TrE   zSkip z in case of z mode.)rG   r   r   rH   r   rI   rJ   )r   r   r   r   r2   rB   r0   rN   r	   APPENDr6   r7   valueZresumable_uploadr   r   r   r   rP   rQ   rM   r8   r   )r!   rW   rX   rY   rZ   r   rS   Z
object_keyrH   existrU   r"   r"   r#   upload   sJ   	
zOssUtilities.uploadN)__name__
__module____qualname__r$   r   r8   staticmethodrB   strr   rV   boolr	   r_   r"   r"   r"   r#   r      s     	


0r   )
__future__r   r   r   Zdatasets.utils.file_utilsr   Zmodelscope.hub.apir   Z.modelscope.msdatasets.download.download_configr   Zmodelscope.utils.config_dsr   Zmodelscope.utils.constantr   r   r	   Zmodelscope.utils.loggerr
   r6   r(   r*   r,   r/   r3   r1   r   r"   r"   r"   r#   <module>   s"   