o
    /ѹgZG                    @   s|  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dlmZ d dlZd dl Z d d	l m!Z! d d
l"m#Z#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 d dl:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZD d dlEmFZF d dlGmHZH d dlImJZJmKZKmLZLmMZMmNZN d dlOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZa d dlbmcZcmdZd d dlemfZf d dlgmhZhmiZimjZjmkZkmlZlmmZmmnZn d dlompZp ef ZqG dd dZrG dd dZsG dd dZtdS )    N)defaultdict)
HTTPStatus)	CookieJar)
expanduserPath)AnyBinaryIODictIterableListOptionalTupleUnion)	urlencode)Session)HTTPAdapterRetrytqdm)API_HTTP_CLIENT_MAX_RETRIESAPI_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_CREDENTIALS_PATHDEFAULT_MAX_WORKERSMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_REQUEST_IDONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODTEMPORARY_FOLDER_NAMEDatasetVisibilityLicensesModelVisibility)
InvalidParameterNotExistErrorNotLoginExceptionRequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper
Repository)add_content_to_fileget_endpointget_readable_folder_sizeget_release_datetimemodel_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORTConfigFieldsDatasetFormationsDatasetMetaFormatsDatasetVisibilityMapDownloadChannelDownloadMode
Frameworks	ModelFileTasksVirgoDatasetConfig)get_file_hashget_file_size)
get_logger)DATASET_LFS_SUFFIXDEFAULT_IGNORE_PATTERNSMODEL_LFS_SUFFIX
CommitInfoCommitOperationCommitOperationAdd	RepoUtils)thread_executorc                   @   sB  e Zd ZdZdeefdee fddZ	ddee fddZ	e
jejdd	fd
edee dee dee dee defddZd
efddZd
efddZefd
edee defddZdddedee defddZedd Ze
jejdd dedddf	d
ed!edee dee dee d"ee d#ee dee dee d$eeee ef  d%eeeee f  fd&d'Z	(	)dd*ed+ee d,ee defd-d.Zdd0eeef defd1d2Z		/dd
ed3ee d0eeef dee fd4d5Z 		/dd
ed3ee d0eeef dee fd6d7Z!d8d9 Z"		dd
ed:ee fd;d<Z#		dd
ed:ee fd=d>Z$	/dd
ed0eeef de%ee ee f fd?d@Z&	/dd
ed0eeef de%ee ee f fdAdBZ'edd/d/i fd
edee dCee dDee d0eeef dEee dee fdFdGZ(ddHdedIedee fdJdKZ)d	eje*jd	fdLedMedee dee dee dNee defdOdPZ+dQdR Z,dLedMefdSdTZ-	/	U	VddWededXedYe.dDef
dZd[Z/	\	(	UddLedMeded]edDed+ed,efd^d_Z0dLedMed`edefdadbZ1edceddefdedfZ2dLedMededdedcedge3fdhdiZ4edje5j6fdkdlZ7e8d/d\fdmedLedMedee dnee doee fdpdqZ9e8fdmedLedMedee fdrdsZ:e8fdLedMedee fdtduZ;e8fdLedMedvedee fdwdxZ<dd`edyedefdzd{Z=dLedMeded|efd}d~Z>dd Z?dedLedMededef
ddZ@dedLedMededef
ddZAdd ZBddLedMededdfddZCdd ZDdedefddZEddeFd	ejddedeeedf dee dee dee dee defddZGdddeddedeHeI d"edee dedee dee deJfddZKdeFddd(dd/ddeeeLeMeNf dededeedf dee d"ee dee dee dee dee deJfddZOdd	dddeFddePed
dedeeeLee eeL f dee d"ee dee deedf dee deeee ef  deeee ef  dedee deJfddZQd/dd(ddededededeeeLeMeNf dee dee dee defddZRdededeeSeeTf  deeSeeTf  fddZU		ddeeeLee eeL f dedeeee ef  deeee ef  deeeVe3f  f
ddZWedeHeI d"edeSeeTf fddZXdS )HubApizModel hub api interface.
    Nendpointc                 C   s   |dur|nt  | _dt i| _t | _t|dddddd}t|d}| j	d	| | j	d
| t
D ]}t| j|tjt| j||d q4t | _dS )u   The ModelScope HubApi。

        Args:
            endpoint (str, optional): The modelscope server http|https address. Defaults to None.
        N
user-agent      )i  i  i  i  F)totalreadconnectbackoff_factorstatus_forcelistrespect_retry_after_header)max_retrieszhttp://zhttps://)timeout)r6   rX   ModelScopeConfigget_user_agentheadersr   sessionr   r   mountr#   setattr	functoolspartialgetattrUploadingCheckupload_checker)selfrX   rc   rb   retryadaptermethod rs   V/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/modelscope/hub/api.py__init__Q   s.   	

zHubApi.__init__access_tokenc                 C   s   |du r
t jd}|sdS | j d}| jj|d|i| | jd}t| |	 }t
| |t t }|j}t| t| t|t t |t t  |t t |fS )a  Login with your SDK access token, which can be obtained from
           https://www.modelscope.cn user center.

        Args:
            access_token (str): user access token on modelscope, set this argument or set `MODELSCOPE_API_TOKEN`.
            If neither of the tokens exist, login will directly return.

        Returns:
            cookies: to authenticate yourself to ModelScope open-api
            git_token: token to access your git repository.

        Note:
            You only have to login once within 30 days.
        NZMODELSCOPE_API_TOKENNNz/api/v1/loginZAccessToken)jsonrf   )osenvirongetrX   rg   postbuilder_headersrf   r0   rx   r1   r   r   cookiesrd   
save_tokensave_cookiessave_user_infor   r   )ro   rv   pathrdtokenr~   rs   rs   rt   loginr   s4   




zHubApi.login model_id
visibilitylicensechinese_nameoriginal_model_idreturnc              
   C   s   |du rt dt }|du rtd| j d}t|\}}	||	||||tjddd}
| j	j
||
|| | jd}t|||
 t|  | j d	| }|S )
a  Create model repo at ModelScope Hub.

        Args:
            model_id (str): The model id
            visibility (int, optional): visibility of the model(1-private, 5-public), default 5.
            license (str, optional): license of the model, default none.
            chinese_name (str, optional): chinese name of the model.
            original_model_id (str, optional): the base model id which this model is trained from

        Returns:
            Name of the model created

        Raises:
            InvalidParameter: If model_id is invalid.
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nzmodel_id is required!)Token does not exist, please login first.z/api/v1/modelsZMODELSCOPE_TRAIN_IDr   )r   NameChineseName
VisibilityLicenseZOriginalModelIdZTrainId)rx   r~   rf   /)r(   rd   get_cookies
ValueErrorrX   r9   ry   rz   r{   rg   r|   r}   rf   r-   r1   rx   )ro   r   r   r   r   r   r~   r   owner_or_groupnamebodyr   Zmodel_repo_urlrs   rs   rt   create_model   s2   	
zHubApi.create_modelc                 C   sZ   t  }|du rtd| j d| }| jj||| | jd}t| t	|
  dS )zDelete model_id from ModelScope.

        Args:
            model_id (str): The model id.

        Raises:
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nr   /api/v1/models/r~   rf   )rd   r   r   rX   rg   deleter}   rf   r0   r1   rx   )ro   r   r~   r   r   rs   rs   rt   delete_model   s   
zHubApi.delete_modelc                 C   s   | j  d| dS )Nr   z.git)rX   )ro   r   rs   rs   rt   get_model_url   s   zHubApi.get_model_urlrevisionc                 C   s   t  }t|\}}|r| j d| d| d| }n| j d| d| }| jj||| | jd}t|t	|| |j
tjkrTt| rL| t S t| t t| dS )an  Get model information at ModelScope

        Args:
            model_id (str): The model id.
            revision (str optional): revision of model.

        Returns:
            The model detail information.

        Raises:
            NotExistError: If the model is not exist, will throw NotExistError

        Note:
            model_id = {owner}/{name}
        r   r   z
?Revision=r   N)rd   r   r9   rX   rg   r{   r}   rf   r.   loggerstatus_coder   OKr/   rx   r   r)   r   r0   )ro   r   r   r~   r   r   r   r   rs   rs   rt   	get_model   s   

zHubApi.get_model	repo_typerepo_idr   c          	      C   s   |dur|  tkrtd| |du s|ddkr!td| t }t|\}}| j d| d| }| jj	||| 
| jd}t|t||d}|d	krQd
S |dkrWdS td| d td| )a  
        Checks if a repository exists on ModelScope

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`, *optional*):
                `None` or `"model"` if getting repository info from a model. Default is `None`.
                TODO: support dataset and studio

        Returns:
            True if the repository exists, False otherwise.
        NzNot support repo-type: %sr   r[   z5Invalid repo_id: %s, must be of format namespace/namer   r   F   Ti  z%Check repo_exists return status code .zOFailed to check existence of repo: %s, make sure you have access authorization.)lowerr@   	Exceptioncountrd   r   r9   rX   rg   r{   r}   rf   r.   r   warn)	ro   r   r   r~   r   r   r   r   coders   rs   rt   repo_exists  s*   

zHubApi.repo_existsc                 C   s`   t j| tj}tjtjtj	t
ji}t|d}t|| W d    d S 1 s)w   Y  d S )Nw)ry   r   joinrI   CONFIGURATIONrB   	frameworkrH   ZtorchtaskrJ   otheropenrx   dump)	model_dircfg_filecfgfilers   rs   rt   _create_default_config<  s   "zHubApi._create_default_configzupload modelr   commit_messagetagignore_file_pattern
lfs_suffixc              
      s&  |du rt d|du rt dtj|rtj|r t dtj|tj}tj|s@t	dtj d| d t
| t }|du rLtdt|}t|}|
du r[g }
t|
trc|
g}
|du sk|du rot d	| |std
|  | j|||||	d tj|t}t }td| d| d td| d zzt||d}||}||vrtd|  ||| ||| t|}|D ]"  d dkrtj| }tj|rt| qtj|dd q|D ]6  d dkr*t fdd|
D r
qtj| }tj |r$t!|tj|  qt"|| q|s<t#j#$ %d}d||f }|durWt|trJ|gn|}|D ]}|&| qN|j'|||d |durj|(|| td| d| d W n	 t)y    w W tj|dd dS tj|dd w )a^
  Upload model from a given directory to given repository. A valid model directory
        must contain a configuration.json file.

        This function upload the files in given directory to given repository. If the
        given repository is not exists in remote, it will automatically create it with
        given visibility, license and chinese_name parameters. If the revision is also
        not exists in remote repository, it will create a new branch for it.

        This function must be called before calling HubApi's login with a valid token
        which can be obtained from ModelScope's website.

        If any error, please upload via git commands.

        Args:
            model_id (str):
                The model id to be uploaded, caller must have write permission for it.
            model_dir(str):
                The Absolute Path of the finetune result.
            visibility(int, optional):
                Visibility of the new created model(1-private, 5-public). If the model is
                not exists in ModelScope, this function will create a new model with this
                visibility and this parameter is required. You can ignore this parameter
                if you make sure the model's existence.
            license(`str`, defaults to `None`):
                License of the new created model(see License). If the model is not exists
                in ModelScope, this function will create a new model with this license
                and this parameter is required. You can ignore this parameter if you
                make sure the model's existence.
            chinese_name(`str`, *optional*, defaults to `None`):
                chinese name of the new created model.
            commit_message(`str`, *optional*, defaults to `None`):
                commit message of the push request.
            tag(`str`, *optional*, defaults to `None`):
                The tag on this commit
            revision (`str`, *optional*, default to DEFAULT_MODEL_REVISION):
                which branch to push. If the branch is not exists, It will create a new
                branch and push to it.
            original_model_id (str, optional): The base model id which this model is trained from
            ignore_file_pattern (`Union[List[str], str]`, optional): The file pattern to ignore uploading
            lfs_suffix (`List[str]`, optional): File types to use LFS to manage. examples: '*.safetensors'.

        Raises:
            InvalidParameter: Parameter invalid.
            NotLoginException: Not login
            ValueError: No configuration.json
            Exception: Create failed.
        Nzmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.zNo z file found in z, creating a default one.zMust login before upload!z5Visibility and License cannot be empty for new model.zCreating new model [%s])r   r   r   r   r   zPushing folder z
 as model r   zTotal folder size z;, this may take a while depending on actual pushing size...)r   Z
clone_fromzCreating new branch %sr   T)ignore_errorsc                    s   g | ]
}t | d uqS N)research).0patternfrs   rt   
<listcomp>      z%HubApi.push_model.<locals>.<listcomp>z%Y_%m_%d_%H_%M_%Sz$[automsg] push model %s to hub at %s)r   Zlocal_branchZremote_branchzSuccessfully push folder z to remote repo [z].)*r(   ry   r   existsisfiler   rI   r   r   warningrW   r   rd   r   r*   listdirr7   
isinstancestrr   infor   r$   r2   r4   Zget_remote_branchesZ
new_branchcheckoutremoveshutilrmtreeanyisdircopytreecopydatetimenowstrftimeZadd_lfs_typepushZtag_and_pushr   )ro   r   r   r   r   r   r   r   r   r   r   r   r   r~   Zfiles_to_saveZfolder_sizetmp_dirZgit_wrapperrepobranchesZfiles_in_reposrcdateZlfs_suffix_listsuffixrs   r   rt   
push_modelF  s   ;







"zHubApi.push_modelr[   
   r   page_number	page_sizec                 C   s   t  }| j d}| jj|d|||f || | jd}t|t|| |j	t
jkr@t| r8| t }|S t| t t| dS )a  List models in owner or group.

        Args:
            owner_or_group(str): owner or group.
            page_number(int, optional): The page number, default: 1
            page_size(int, optional): The page size, default: 10

        Raises:
            RequestError: The request error.

        Returns:
            dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group}
        r   z.{"Path":"%s", "PageNumber":%s, "PageSize": %s})datar~   rf   N)rd   r   rX   rg   putr}   rf   r.   r   r   r   r   r/   rx   r   r+   r   r0   )ro   r   r   r   r~   r   r   r   rs   rs   rt   list_models  s$   
zHubApi.list_modelsFuse_cookiesc                 C   s6   d }t |tr|}|S |rt }|d u rtd|S )Nr   )r   r   rd   r   r   )ro   r   r~   rs   rs   rt   _check_cookie  s   
zHubApi._check_cookiecutoff_timestampc                 C   s.   | j |||d}|rdd |D }|S g }|S )3  Get model branch and tags.

        Args:
            model_id (str): The model id
            cutoff_timestamp (int): Tags created before the cutoff will be included.
                                    The timestamp is represented by the seconds elapsed from the epoch time.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        )r   r   r   c                 S      g | ]}|d  qS Revisionrs   r   xrs   rs   rt   r         z/HubApi.list_model_revisions.<locals>.<listcomp>N)list_model_revisions_detail)ro   r   r   r   Ztags_detailstagsrs   rs   rt   list_model_revisions  s   zHubApi.list_model_revisionsc           	      C   sz   |  |}|du rt }| j d| d| }| jj||| | jd}t|t|| |	 }t
| |t }|d d S )r   Nr   z/revisions?EndTime=%sr   RevisionMapTags)r   r8   rX   rg   r{   r}   rf   r.   r   rx   r1   r   )	ro   r   r   r   r~   r   r   r   r   rs   rs   rt   r     s   


z"HubApi.list_model_revisions_detailc                 C   s"   |D ]}|d |kr|  S qd S )Nr   rs   )ro   detailsr   itemrs   rs   rt   get_branch_tag_detail4  s
   zHubApi.get_branch_tag_detailr~   c                    s8  t   tttj  }| j||d u rdn|d\}}|r&dd |D ng }|r1dd |D ng } |t kro|d u rFt}t	
d|  ||vrV||vrVtd||f | ||}	|	d u rf| ||}	t	d|  |	S |d ur||v r| ||}	t	d	|  |	S t|d
kr|d u s|tkrt}ntd||f | ||}	|	S |d u r|r fdd|D ng }
t|
d
kr|
d
 d }|
d
 }	nt}| ||}	dd| }t	d|  t	d|  |	S ||vr|tkrt	d | ||}	ndd| }td|||f | ||}	t	
d|  |	S )NFr   c                 S   r   r   rs   r   rs   rs   rt   r   E  r   z4HubApi.get_valid_revision_detail.<locals>.<listcomp>c                 S   r   r   rs   r   rs   rs   rt   r   F  r   z:Model revision not specified, using default: [%s] version.z$The model: %s has no revision : %s .z!Development mode use revision: %sz9Using branch: %s as version is unstable, use with cautionr   z#The model: %s has no revision: %s !c                    s   g | ]
}|d   kr|qS )Z	CreatedAtrs   r   Zrelease_timestamprs   rt   r   b  s    

r   z[%s],z5Model revision should be specified from revisions: %sz.Model revision not specified, use revision: %sz?Using the master branch is fragile, please use it with caution!z0The model: %s has no revision: %s valid are: %s!z%Use user-specified model revision: %s)r8   introundr   r   	timestamp#get_model_branches_and_tags_detailsr"   r=   r   r   r)   r   debugr   lenr   )ro   r   r   r~   Zcurrent_timestampZall_branches_detailZall_tags_detailZall_branchesZall_tagsZrevision_detailZrevisions_detailZvlrs   r   rt   get_valid_revision_detail:  sv   
(


z HubApi.get_valid_revision_detailc                 C   s   | j |||dd S )N)r   r   r~   r   )r  )ro   r   r   r~   rs   rs   rt   get_valid_revision}  s   zHubApi.get_valid_revisionc                 C   st   |  |}| j d| d}| jj||| | jd}t|t|| | }t	| |t
 }|d d |d d fS )q  Get model branch and tags.

        Args:
            model_id (str): The model id
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        r   z
/revisionsr   r   ZBranchesr   N)r   rX   rg   r{   r}   rf   r.   r   rx   r1   r   )ro   r   r   r~   r   r   r   r   rs   rs   rt   r    s   


z*HubApi.get_model_branches_and_tags_detailsc                 C   sF   | j ||d\}}|rdd |D ng }|rdd |D ng }||fS )r  )r   r   c                 S   r   r   rs   r   rs   rs   rt   r     r   z6HubApi.get_model_branches_and_tags.<locals>.<listcomp>c                 S   r   r   rs   r   rs   rs   rt   r     r   N)r  )ro   r   r   Zbranches_detailZtags_detailr   r   rs   rs   rt   get_model_branches_and_tags  s   
z"HubApi.get_model_branches_and_tagsroot	recursiverf   c                 C   s   |rd| j |||f }nd| j ||f }| |}|dur$|d|  }|du r+| jn|}tt j|d< | jj|||d}	t	|	t
|| |	 }
t|
 g }|
t d D ]}|d d	ksd|d d
kreqV|| qV|S )a`  List the models files.

        Args:
            model_id (str): The model id
            revision (Optional[str], optional): The branch or tag name.
            root (Optional[str], optional): The root path. Defaults to None.
            recursive (Optional[str], optional): Is recursive list files. Defaults to False.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            headers: request headers

        Returns:
            List[dict]: Model file list.
        z7%s/api/v1/models/%s/repo/files?Revision=%s&Recursive=%sz+%s/api/v1/models/%s/repo/files?Recursive=%sNz&Root=zX-Request-IDr   Filesr   z
.gitignorez.gitattributes)rX   r   rf   r   uuiduuid4hexrg   r{   r.   r   rx   r1   r   append)ro   r   r   r	  r
  r   rf   r   r~   r   r   filesr   rs   rs   rt   get_model_files  s0   

zHubApi.get_model_files)r   filenamec                C   s&   | j |d|d}dd |D }||v S )ap  Get if the specified file exists

        Args:
            repo_id (`str`): The repo id to use
            filename (`str`): The queried filename, if the file exists in a sub folder,
                please pass <sub-folder-name>/<file-name>
            revision (`Optional[str]`): The repo revision
        Returns:
            The query result in bool value
        T)r
  r   c                 S   r   r   rs   )r   r   rs   rs   rt   r     r   z&HubApi.file_exists.<locals>.<listcomp>N)r  )ro   r   r  r   r  rs   rs   rt   file_exists  s   zHubApi.file_existsdataset_name	namespacedescriptionc                 C   s   |d u s|d u rt dt }|d u rtd| j d}d |fd |fd |fd |fd |fd |fd}	| jj||	|| | jd}
t	|
||	 t
|
  | j d| d| }td|  |S )	Nz(dataset_name and namespace are required!r   /api/v1/datasets)r   r   ZOwnerr   r   Description)r  r~   rf   
/datasets/r   zCreate dataset success: )r(   rd   r   r   rX   rg   r|   r}   rf   r-   r1   rx   r   r   )ro   r  r  r   r   r   r  r~   r   r  r   Zdataset_repo_urlrs   rs   rt   create_dataset  s0   	
zHubApi.create_datasetc                 C   sL   | j  d}i }| jj||| | jd}t| | t }dd |D S )Nr  )paramsrf   c                 S   r   )r   rs   r   rs   rs   rt   r   '  r   z(HubApi.list_datasets.<locals>.<listcomp>)rX   rg   r{   r}   rf   r0   rx   r   )ro   r   r  r   Zdataset_listrs   rs   rt   list_datasets   s   

zHubApi.list_datasetsc           	      C   sb   | j  d| d| }t }| jj||d}| }t||| |d d }|d d }||fS )z Get the dataset id and type. /api/v1/datasets/r   r~   DataZIdTypeNrX   rd   r   rg   r{   rx   r,   )	ro   r  r  datahub_urlr~   r   resp
dataset_iddataset_typers   rs   rt   get_dataset_id_and_type)  s   zHubApi.get_dataset_id_and_typed   Truedataset_hub_idfiles_metadatarc   c                 C   s^   | j  d| d}|d|d}t }|rd|d< | jj||||d}	|	 }
t||
|	 |
S )z$
        Get dataset infos.
        r  
/repo/treeN)r   Root	RecursiveTZblobs)r  r~   rc   r!  )ro   r)  r   r*  rc   r
  r"  r  r~   r   r#  rs   rs   rt   get_dataset_infos4  s   	zHubApi.get_dataset_infosT	root_pathc                 C   s   | j ||d\}}	|rdnd}| j d| d}
|r|nd|r!|nd|||d}t }| jj|
||d	}| }t|
|| |S )
N)r  r  r(  Falser  r+  Zmasterr   )r   r,  r-  Z
PageNumberZPageSize)r  r~   )r&  rX   rd   r   rg   r{   rx   r,   )ro   r  r  r   r/  r
  r   r   r)  r%  r"  r  r~   r   r#  rs   rs   rt   list_repo_treeH  s   	

zHubApi.list_repo_treer$  c           
      C   s   | j  d| d| }t }| jj||| | jd}| }t||| |d }	|	du r<t	d| d| d| d	|	d
 }	|	S )z( Get the meta file-list of the dataset. r  z/repo/tree?Revision=r   r  Nz'The modelscope dataset [dataset_name = z, namespace = z, version = z] dose not existr  )
rX   rd   r   rg   r{   r}   rf   rx   r,   r)   )
ro   r  r  r$  r   r"  r~   r   r#  	file_listrs   rs   rt   get_dataset_meta_file_lista  s"   
z!HubApi.get_dataset_meta_file_listr%  meta_cache_dirc                 C   sX   t j|t|  tjj }t|d}|d W d   dS 1 s%w   Y  dS )z
        Dump the data_type as a local file, in order to get the dataset
         formation without calling the datahub.
        More details, please refer to the class
        `modelscope.utils.constant.DatasetFormations`.
        r   z3*** Automatically-generated file, do not modify ***N)	ry   r   r   r   rC   Zformation_mark_extvaluer   write)r%  r4  Zdataset_type_file_pathfprs   rs   rt   dump_datatype_files  s   "zHubApi.dump_datatype_filer2  c              
   C   s   t t}t|}t| }	t }
tj||d |D ]r}|d }tj	
|d }||	v r| j d| d| d| d| 	}| jj||
d}t| tj	||}tj	|ritd	| d
| d || | qt|d}||j W d    n1 sw   Y  || | q||fS )N)r%  r4  r   r  r   /repo?Revision=
&FilePath=r  zReusing dataset z's python file ()wb)r   listrC   rD   rd   r   rW   r8  ry   r   splitextrX   rg   r{   r0   r   r   r   r   r  r   r6  content)ro   r  r  r   r4  r%  r2  Zlocal_pathsZdataset_formationZdataset_meta_formatr~   	file_info	file_path	extensionr"  r   Z
local_pathr   rs   rs   rt   "get_dataset_meta_files_local_paths  s:   z)HubApi.get_dataset_meta_files_local_paths   c                    s  ddl }ddlm} ddl}tj||| jdd	 }|t
jkr.tj|r.t| tj|r>td|  |S t }td tj| |dd	}t|jd
d}	||	dd}
 fdd}d}t|dQ}||D ]D}|
t| | drdd |D }t|dkrqp|dkrd}nd}||}|j|d|dd |d7 }qp|D ]	}||d  qqpW d   n1 sw   Y  |
  |S )zO
        Fetch the meta-data files from the url, e.g. csv/jsonl files.
        r   Nr   zUTF-8encodingzReusing cached meta-data file: zLoading meta-data file ...T)r~   streamzcontent-length)r\   Zdynamic_ncolsc                 3   sH    g }|   D ]}|d}|| t| kr|V  g }q|V  d S )Nutf-8)
iter_linesdecoder  r  )r#  Z
chunk_datar   
chunk_sizers   rt   	get_chunk  s   


z3HubApi.fetch_meta_files_from_url.<locals>.get_chunkaZjsonlc                 S   s   g | ]}|  rt|qS rs   )striprx   loads)r   liners   rs   rt   r     s    z4HubApi.fetch_meta_files_from_url.<locals>.<listcomp>F\)indexheader
escapecharr[   
)hashlib	tqdm.autor   Zpandasry   r   r   md5encode	hexdigestrG   ZFORCE_REDOWNLOADr   r   r   r   rd   r   requestsr{   r   rf   r   updater  endswithZ	DataFrameZto_csvr6  close)urlZout_pathrM  moderX  r   pdr~   response
total_sizeprogressrN  Ziter_numr   chunkZwith_headerZchunk_dfrR  rs   rL  rt   fetch_meta_files_from_url  sH    





z HubApi.fetch_meta_files_from_url	file_nameviewextension_filterc           	      C   sJ   |r|r|s
t dd|||d}t|}| j d| d| d| }|S )Nz:Args (file_name, dataset_name, namespace) cannot be empty!ZSDK)ZSourcer   ZFilePathZViewr  r   /repo?)r   r   rX   )	ro   ri  r  r  r   rj  rk  r  file_urlrs   rs   rt   get_dataset_file_url  s   	zHubApi.get_dataset_file_urlc              	   C   s>   |rt j|d tv r| j d| d| d| d| 	}|S )Nr9  r  r   r:  r;  )ry   r   r?  r>   rX   )ro   ri  r  r  r   rs   rs   rt   get_dataset_file_url_origin  s   z"HubApi.get_dataset_file_url_originc                 C   s&   | j  d| d| d| }| |S )Nr  r   /ststoken?Revision=)rX   datahub_remote_call)ro   r  r  r   r"  rs   rs   rt   get_dataset_access_config  s   
z HubApi.get_dataset_access_configcheck_cookiec           	      C   sh   | j  d| d| d| }|r| jdd}nt }| jj||| | jd}| }t	| |d S )Nr  r   rp  Tr   ra  r~   rf   r  )
rX   r   rd   r   rg   r{   r}   rf   rx   r1   )	ro   r  r  rs  r   r"  r~   r   r#  rs   rs   rt   !get_dataset_access_config_session
  s   
z(HubApi.get_dataset_access_config_sessionversionc           
      C   s   t jtjd}|stdtj | d}tjt	
 }t||d}t|d}| jj|||| | jdd}| }	|	d d	krJtd
|	 |	d S )z.
        Get virgo dataset meta info.
        r   z"Virgo endpoint is not set in env: z/data/set/download)Z	dataSetIdZdataSetVersion)r   i  )ra  rx   r~   rf   rc   r   r   zFailed to get virgo dataset: r   N)ry   rz   r{   rK   Zenv_virgo_endpointRuntimeErrorr]  utilsdict_from_cookiejarrd   r   dictrg   r|   r}   rf   rx   )
ro   r$  rv  Zvirgo_endpointZvirgo_dataset_urlr~   Zdataset_infor   r   r#  rs   rs   rt   get_virgo_meta   s,   

zHubApi.get_virgo_metazip_file_namec                 C   s   | j  d| d| }t }| jj||| | jd}| }t| |d }	t	|	d }
| d| }| jj||| | jd}| }t| |d }|
d d | d | d | }||d	< |S )
Nr  r   rt  r  r   rp  z	-unzipped_ZDir)
rX   rd   r   rg   r{   r}   rf   rx   r1   rE   )ro   r  r  r   r|  r"  r~   r   r#  r   r   Zdatahub_sts_urlZr_stsZresp_stsZdata_stsZfile_dirrs   rs   rt   &get_dataset_access_config_for_unzipped=  s&   



 z-HubApi.get_dataset_access_config_for_unzippedc           
      C   sd   | j  d| d| d| d| d| d| }t }| jj||dd}	|	 }	t|	 |	d	 }	|	S )
Nr  r   z/oss/tree/?MaxLimit=
&Revision=z&Recursive=z&FilterDir=i  )ra  r~   rc   r  )rX   rd   r   rg   r{   rx   r1   )
ro   r  r  Z	max_limitZis_recursiveZis_filter_dirr   ra  r~   r#  rs   rs   rt   list_oss_dataset_objectsV  s   zHubApi.list_oss_dataset_objectsobject_namec              	   C   n   |r|r|r|st d| j d| d| d| d| 	}t }| jj||d}| }t| |d }|S )NArgs cannot be empty!r  r   z
/oss?Path=r  ra  r~   Messager   rX   rd   r   rg   r   rx   r1   ro   r  r  r  r   ra  r~   r#  rs   rs   rt   delete_oss_dataset_objectb  s   "z HubApi.delete_oss_dataset_objectc              	   C   r  )Nr  r  r   z/oss/prefix?Prefix=z/&Revision=r  r  r  r  rs   rs   rt   delete_oss_dataset_dirp  s   zHubApi.delete_oss_dataset_dirc                 C   s>   t  }| jj||dt  id}| }t||| |d S )NrY   r   r  )rd   r   rg   r{   re   rx   r,   )ro   ra  r~   r   r#  rs   rs   rt   rq    s   
zHubApi.datahub_remote_calluse_streamingc              
   C   s(  t ddk}|r|r|s|szdt }| j d| d| d}| jj||| | jd}t	| t
jj}d}	tt jv rAt jt }tt jv rKt jt }	| j d| d| d| d	|	 	}
| jj|
|| | jd}| }t| W d S  ty } zt| W Y d }~d S d }~ww d S d S d S d S )
NZCI_TESTr(  r  r   z/download/increaser   r   z/download/uv/z?user=)ry   getenvrd   r   rX   rg   r|   r}   rf   r0   rF   ZLOCALr5  r   rz   r    rx   r1   r   r   error)ro   r  r  r  Z
is_ci_testr~   Zdownload_count_urlZdownload_count_respZchannel	user_nameZdownload_uv_urlZdownload_uv_respers   rs   rt   dataset_download_statistics  s<   







z"HubApi.dataset_download_statisticsc                 C   s   t tt ji|S r   )r!   r   r  r  r  )ro   rf   rs   rs   rt   r}     s   zHubApi.builder_headersc                 C   s&   | d\}}| j d| d| dS )Nr   r  rl  )splitrX   )ro   r   
_namespaceZ_dataset_namers   rs   rt   get_file_base_path  s   zHubApi.get_file_base_pathpublic)r   r   r   r   r   r   c                K   s  |st d| j|d |d}t|dkrt d|\}	}
|tkrdd tj D }||	 }|d u r@t d| d	| j
||d
s| j||||d}t 5}ddlm} |||}dddd}|d}|soi }i ||}t|dt|gdd W d    |S 1 sw   Y  |S | j d| }|S |tkrdd tj D }||	 }|d u rt d| d	| j
||d
s| j|
|	|||d}|S | j d|	 d|
 }|S t d| dt )NzRepo id cannot be empty!rv   r   rZ   zBInvalid repo id, should be in the format of `owner_name/repo_name`c                 S       i | ]\}}| d s||qS __
startswithr   kvrs   rs   rt   
<dictcomp>       z&HubApi.create_repo.<locals>.<dictcomp>zInvalid visibility: z9, supported visibilities: `public`, `private`, `internal`r   )r   r   r   r   r   r3   Zpytorchztext-generationT)r   r   Zallow_remoteconfig_jsonzconfiguration.json)Zignore_push_errorc                 S   r  r  r  r  rs   rs   rt   r    r  )r  r  r   r   r   r  Invalid repo type: , supported repos: )r   r   r  r  r@   r'   __dict__itemsr{   upperr   r   tempfileTemporaryDirectorymodelscope.hub.repositoryr4   r5   rx   dumpsrX   r?   r%   r  rA   )ro   r   r   r   r   r   r   kwargsZrepo_id_listr  	repo_nameZvisibilitiesrepo_urlZtemp_cache_dirr4   r   Zdefault_configr  configrs   rs   rt   create_repo  st   




''zHubApi.create_repo)commit_descriptionr   r   r   
operationsr  c                C   s   | j  d| d| d| }|pd| }|pd}| j|d | j||d}	t }
|
d u r2tdtj|| | j	t
|	|
d	}|
 }|d
 sT|d }t|  t|||ddS )N/api/v1/repos/s/z/commit/z
Commit to r   r  )r  r   r   rf   r   r~   ZSuccessr  )Z
commit_urlr   r  oid)rX   r   _prepare_commit_payloadrd   r   r   r]  r|   r}   rf   rx   r  r   r   rR   )ro   r   r  r   r  r   r   r   ra  payloadr~   rd  r#  rs   rs   rt   create_commit  s6   
zHubApi.create_commitz[Uploading])r   r   r   r  buffer_size_mb	tqdm_descdisable_tqdmpath_or_fileobjpath_in_repor  r  r  c       
      	   C   sb  |t vrtd| dt  |stdt|ttfr.tjtj|}|p,tj	|}n|s4tdt|t
jr>| }| j| | jj|g|d | j|d |d urY|nd| d}|d	krgtd
t||d}|d }|d }| j||||||
|	d}t|||d}| j||rdnd|_|d |_|g}td| d | j||||||d}|S )Nr  r  z$Path or file object cannot be empty!z#Arg `path_in_repo` cannot be empty!file_path_listr   r  zUpload z to ModelScope hubr   z4Buffer size: `buffer_size_mb` must be greater than 0)file_path_or_objr  	file_size	file_hashr   r   sha256sizer   r  r  r  r  file_hash_infolfsnormalis_uploadedzCommitting file to  ...)r   r  r   r  r   r   )rA   r   r   r   r   ry   r   abspathr   basenameioBufferedIOBaser]   rn   
check_filecheck_normal_filesr   rL   _upload_blobrT   is_lfs_upload_mode_is_uploadedprintr  )ro   r  r  r   r   r   r   r  r  r  r  hash_info_dr  r  
upload_resZadd_operationr  commit_infors   rs   rt   upload_file,  sl   
	zHubApi.upload_file)
folder_pathr  r   r  r   r   allow_patternsignore_patternsmax_workersr   r  r  r  r  c             
      sv  t vrtd dt  |r|nd }|	r|	nd }	|	d u r!g }	nt|	tr)|	g}	|	t7 }	j|d |d ur9|nd  d}|pBd}j||||	d}jjdd	 |D d
 t	|
dd fdd}|| |||ddd}g }|D ]-}|d }|d }|d }|d }t
|||d}j|rdnd|_||_|| qwtd  d j |||||d}|S )Nr  r  r  z
Upload to z on ModelScope hubzUploading files)folder_path_or_filesr  r  r  c                 S   s   g | ]\}}|qS rs   rs   )r   r}  r   rs   rs   rt   r     r   z(HubApi.upload_folder.<locals>.<listcomp>r  F)r  r  c              	      s`   | \}}t |d}|d }|d }j ||||dkrdndd| d d	}|||d
 |dS )N)r  r  r    P FTz[Uploading ]r  r  )file_path_in_reporB  r  r  )rL   r  )Z	item_pairr  r  rB  r  r  r  r  r   r   ro   rs   rt   _upload_items  s(   
z+HubApi.upload_folder.<locals>._upload_itemsr[   )r   r   r   r   r  r  r  r  rB  r  r  r  r  r  zCommitting folder to r  )r   r  r   r  r   r   r   )rA   r   r   r   rP   r   _prepare_upload_folderrn   r  rV   rT   r  r  r  r  r  r  )ro   r   r  r  r   r  r   r   r  r  r  r   prepared_repo_objectsr  Zuploaded_items_listr  Zitem_dZprepared_path_in_repoZprepared_file_pathr  r  optr  rs   r  rt   upload_folder  sx   


zHubApi.upload_folder)r  r  r  r  r  r   c             	   C   s  t d dd d d}	||dg}
| j|||
d}t|dkr |d nd }|d u r9td|d d  d	 d
|	d< |	S t }|rCt |nd }|d u rMtd| j	dd|d  i | 
| j}|d d fdd}t|dd
||d`}t|ttfrt|d}tj|d ||||d}W d    n71 sw   Y  n.t|trtj|d ||t||d}nt|tjrtj|d ||||d}ntdW d    n1 sw   Y  | }t| |d |	d< |d |	d< |d |	d< |	S )NF)ra  r  r   
status_msg)r  r  )r   r   objectsr[   r   zBlob    z  has already uploaded, reuse it.Tr  r   Cookiezm_session_id=m_session_idrE  c                 s   s,    	 |  |}|sdS |t| |V  q)z8Lazy function (generator) to read a file piece by piece.TN)r]   r^  r  )Zfile_objectpbarrM  Zckrs   rs   rt   read_in_chunks&  s   
z+HubApi._upload_blob.<locals>.read_in_chunksB)r\   unitZ
unit_scaledescdisablerbra  )rf   r   zInvalid data type to uploadZCoder   r  r  )rz  _validate_blobr  r   r   rd   r   r   rf   r^  r}   r   r   r   r   r   r]  r   bytesr  BytesIOr  rx   r1   )ro   r   r   r  r  r   r  r  r  Zres_dr  upload_objectsZupload_objectr~   rf   r  r  r   rd  r#  rs   rs   rt   r    sz   	
!zHubApi._upload_blobr  c                C   s   | j  d| d| d}d|d}t }|du rtdtj|| | jt	||d}| }t
| g }	|d	 d
 }
|
D ]}|	|d d d |d d q?|	S )a  
        Check the blob has already uploaded.
        True -- uploaded; False -- not uploaded.

        Args:
            repo_id (str): The repo id ModelScope.
            repo_type (str): The repo type. `dataset`, `model`, etc.
            objects (List[Dict[str, Any]]): The objects to check.
                oid (str): The sha256 hash value.
                size (int): The size of the blob.

        Returns:
            List[Dict[str, Any]]: The result of the check.
        r  r  z/info/lfs/objects/batchupload)	operationr  Nr   r  r  r  actionshrefr  )ra  r  )rX   rd   r   r   r]  r|   r}   rf   rx   r  r1   r  )ro   r   r   r  ra  r  r~   rd  r#  r  Zresp_objectsobjrs   rs   rt   r  Y  s0   
zHubApi._validate_blobr  c           	         s   d  d }t |trtj|d r|}ntdtj|r"|g}n| |d u rQ| j  t 	 
    sBtd  d fddt dD ni |D ]}tj|rk| j| |tj|< qUttj ||d}|r|d	 d	nd
fdd|D }|S )Nr   z0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directoryc                    s$   i | ]}|  r|  |qS rs   )is_filerelative_toas_posix)r   r   )r  rs   rt   r    s    z1HubApi._prepare_upload_folder.<locals>.<dictcomp>z**/*)r  r  r   r   c                    s    g | ]} | t | fqS rs   )r   )r   relpath)prefixrelpath_to_abspathrs   rt   r     s    z1HubApi._prepare_upload_folder.<locals>.<listcomp>)r   r>  ry   r   r   r   rn   check_folderr   r   resolveis_dirsortedglobr  r  rU   Zfilter_repo_objectskeysrP  )	ro   r  r  r  r  Z
files_pathr   Zfiltered_repo_objectsr  rs   )r  r   r  rt   r    sB   


zHubApi._prepare_upload_folderc              	   C   s  |g d}d}| D ]t}t |tr"|jr"td|j d |d7 }q	t |trJ|jdkrJ|jr1dnd|jd|jj	d	|
  d
d}|d | q	t |trp|jdkrp|jrYdnd|jd|jj	|jjd	d	d}|d | q	td| dt|dd |dkrtd| d |S )zN
        Prepare the commit payload to be sent to the ModelScope hub.
        )r   r  r   zSkipping file 'z(' in commit (ignored by gitignore file).r[   r  r^  creater   base64)actionr   typer  r  r@  rG  r  r  z(Unknown operation to commit. Operation: z. Upload mode: r  NzSkipped z/ file(s) in commit (ignored by gitignore file).)r   rT   Z_should_ignorer   r  r  r  r  Zupload_infor  Z
b64contentrK  r  r  r   rl   r   )r  r   r  Znb_ignored_filesr  Zcommit_actionrs   rs   rt   r    sJ   

		
zHubApi._prepare_commit_payloadr   )r[   r   )F)NFrw   )Fr'  r(  )Tr[   r'  )r[   )Y__name__
__module____qualname____doc__r   r   r   r   ru   r   r'   ZPUBLICr&   Z	APACHE_V2r   r   r   r   r;   r   boolr   staticmethodr   r<   r   r   r   rz  r   r   r   r   r   r   r  r  r   r  r  r  r  r%   r  r  r&  floatr.  r1  r3  r8  r>  rD  rG   ZREUSE_DATASET_IF_EXISTSrh  r:   rn  ro  rr  ru  r{  r~  r  r  r  rq  r  r}   r  r@   r  r   rS   rR   r  r   r  r	   r  r   r  r  r
   r   r  tupler  r  rs   rs   rs   rt   rW   M   s   
#
.
4
*
*
	

 
$




E






4

&	


!<
"








	

S	

6
	

]
	

~	

^
:
7
rW   c                   @   s   e Zd ZeeZdZdZdZdZ	dZ
edd Zedefdd	Zed
d Zedd ZedefddZededefddZedeeef fddZedee fddZeddeeedf defddZdS )rd   r~   Z	git_tokenuserrg   Fc                   C   s   t jtjdd d S )NT)exist_ok)ry   makedirsrd   path_credentialrs   rs   rs   rt   make_sure_credential_path_exist  s   z0ModelScopeConfig.make_sure_credential_path_existc                 C   sR   t   ttjt jt jd}t	| | W d    d S 1 s"w   Y  d S )Nzwb+)
rd   r  r   ry   r   r   r  COOKIES_FILE_NAMEpickler   )r~   r   rs   rs   rt   r     s   
"zModelScopeConfig.save_cookiesc                  C   s   t jtjtj} t j| rMt| d1}t	|}|D ]}|j
dkr;| r;tjs;dt_td  W d    d S q|W  d    S 1 sHw   Y  d S )Nr  r  Tz[Authentication has expired, please re-login for uploading or accessing controlled entities.)ry   r   r   rd   r  r  r   r   r  loadr   
is_expiredcookie_expired_warningr   r   )Zcookies_pathr   r~   cookiers   rs   rt   r     s$   


	 
zModelScopeConfig.get_cookiesc                  C   s   t jtjtj} d}t j| r2t| d}t|	 
 dd}|W  d    S 1 s-w   Y  |dks<t|dkrbtt j}t  t| d}|| W d    |S 1 s]w   Y  |S )Nr   r  rI  rF      w+)ry   r   r   rd   r  USER_SESSION_ID_FILE_NAMEr   r   r   readlinerP  r  r  r  r  r  r6  )Zsession_pathZ
session_idr   Zwfrs   rs   rt   get_user_session_id,  s$   
 
z$ModelScopeConfig.get_user_session_idr   c                 C   sP   t   ttjt jt jd}||  W d    d S 1 s!w   Y  d S )Nr   )	rd   r  r   ry   r   r   r  GIT_TOKEN_FILE_NAMEr6  r   r   rs   rs   rt   r   =  s   
"zModelScopeConfig.save_tokenr  
user_emailc                 C   sX   t   ttjt jt jd}|d| |f  W d    d S 1 s%w   Y  d S )Nr   z%s:%s)	rd   r  r   ry   r   r   r  USER_INFO_FILE_NAMEr6  )r  r&  r   rs   rs   rt   r   E  s   
"zModelScopeConfig.save_user_infor   c                  C   s~   z4t tjtjtjddd} |  }|dd |dd fW  d    W S 1 s-w   Y  W dS  t	y>   Y dS w )Nr   rI  rF  :r   r[   rw   )
r   ry   r   r   rd   r  r'  r]   r  FileNotFoundError)r   r   rs   rs   rt   get_user_infoM  s"   
$	zModelScopeConfig.get_user_infoc                  C   sh   d} z't tjtjtjddd}| } W d   W | S 1 s"w   Y  W | S  ty3   Y | S w )z
        Get token or None if not existent.

        Returns:
            `str` or `None`: The token, `None` if it doesn't exist.

        Nr   rI  rF  )	r   ry   r   r   rd   r  r$  r]   r)  r%  rs   rs   rt   	get_token[  s&   	

zModelScopeConfig.get_tokenN
user_agentc                 C   s   d}t tjv rtjt  }d}ttjv rtjt }ddlm} d|t t	 t t
 ||f }t| trH|dddd |  D  7 }|S t| trS|d|  7 }|S )
a  Formats a user-agent string with basic info about a request.

        Args:
            user_agent (`str`, `dict`, *optional*):
                The user agent info in the form of a dictionary or a single string.

        Returns:
            The formatted user-agent string.
        Zcustomunknownr   )__version__zSmodelscope/%s; python/%s; session_id/%s; platform/%s; processor/%s; env/%s; user/%sz; c                 s   s"    | ]\}}| d | V  qdS )r   Nrs   r  rs   rs   rt   	<genexpr>  s     z2ModelScopeConfig.get_user_agent.<locals>.<genexpr>N)r   ry   rz   r    Z
modelscoper.  platformpython_versionrd   r#  	processorr   rz  r   r  r   )r,  envr  r.  Zuars   rs   rt   re   p  s,   




	 
zModelScopeConfig.get_user_agentr   )r  r  r  r   r   r  r  r$  r'  r!  r  r  r  r   r   r   r#  r   r   r   r   r*  r   r+  r   r
   re   rs   rs   rs   rt   rd     s2    


$rd   c                   @   s   e Zd Z					ddededed	ed
ef
ddZdd Zdeeef fddZ	deeee
ef dedefddZdeeeef  deddfddZdS )rm   順 '         2 r    @max_file_countmax_file_count_in_dirmax_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitc                 C   s"   || _ || _|| _|| _|| _d S r   )r8  r9  r:  r;  r<  )ro   r8  r9  r:  r;  r<  rs   rs   rt   ru     s
   
zUploadingCheck.__init__c                 C   sf   t |ttfrtj|std| dt|}|| jkr1td| jd  dt	|d d dd S )NFile  does not existzFile exceeds size limit:    @z	 GB, got     GB)
r   r   r   ry   r   r   r   rM   r:  r   )ro   r  r  rs   rs   rt   r    s   

zUploadingCheck.check_filer  c                 C   s   d}d}t |trt|}| D ]Z}| r=|d7 }t|}|| jkr<td| d| jd  ddt|d d dq|	 rk|d7 }| 
|\}}|| | jkrctd	| d
||  d| j ||7 }||7 }q|| jkr|td| d| j ||fS )Nr   r[   r=  z exceeds size limit: r?  rA  zgot r@  z
Directory z
 contains z items and exceeds limit: zTotal file count z and exceeds limit: )r   r   r   iterdirr  rM   r:  r   r   r  r  r9  r8  )ro   r  Z
file_countZ	dir_countr   Z	item_sizeZsub_file_countZsub_dir_countrs   rs   rt   r    s4   


zUploadingCheck.check_folderr  r   r   c                 C   s   d}t |ttfr;t|}| std| d|tkr%|jtvr$d}n|tkr1|jt	vr0d}n
td| dt
 t|}|| jkpE|S )NTr=  r>  Fr  r  )r   r   r   r   r   r@   r   rQ   r?   rO   rA   rM   r;  )ro   r  r   Zhit_lfs_suffixr  rs   rs   rt   r    s    

zUploadingCheck.is_lfsr  Nc                    sT    fdd|D }t dd |D }|jkr(td|d  djd  dd S )Nc                    s   g | ]
} | s|qS rs   )r  r   r   r   ro   rs   rt   r     r   z5UploadingCheck.check_normal_files.<locals>.<listcomp>c                 S   s   g | ]}t |qS rs   )rM   rC  rs   rs   rt   r     r   zTotal size of non-lfs files i   zMB and exceeds limit: ZMB)sumr<  r   )ro   r  r   Znormal_file_listre  rs   rD  rt   r    s   

z!UploadingCheck.check_normal_files)r4  r5  r6  r  r7  )r  r  r  r   ru   r  r   r   r   r  r  r	   r  r  r   r  rs   rs   rs   rt   rm     s*    
"&rm   )ur   rj   r  ry   r  r0  r   r   r  r  collectionsr   httpr   http.cookiejarr   os.pathr   pathlibr   typingr   r	   r
   r   r   r   r   r   urllib.parser   rx   r]  r   Zrequests.adaptersr   r   rY  r   Zmodelscope.hub.constantsr   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   Zmodelscope.hub.errorsr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   Zmodelscope.hub.gitr2   r  r4   Zmodelscope.hub.utils.utilsr5   r6   r7   r8   r9   Zmodelscope.utils.constantr:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   Zmodelscope.utils.file_utilsrL   rM   Zmodelscope.utils.loggerrN   Zmodelscope.utils.repo_utilsrO   rP   rQ   rR   rS   rT   rU   Zmodelscope.utils.thread_utilsrV   r   rW   rd   rm   rs   rs   rs   rt   <module>   sd   (P0P
$             G 