o
    /ѹgo                     @   sx  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZ d dlZd dlmZ d dlmZ d d	lmZmZ d d
lmZmZmZm Z m!Z!m"Z"m#Z# d dl$m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+m,Z, d dl-m.Z. ddl/m0Z0m1Z1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7m8Z8 e. Z9e&dddddfde:de:dee: dee: deee:df dee; dee dee: dee: fddZ<e%dddddfde:de:dee: dee:edf dee: deeee:f  dee; dee de:fdd Z=de&ddddddd!d"e:de:d#e:dee: dee: deee:df dee; dee dee: d$e;dee: fd%d&Z>de:de:fd'd(Z?dde(fde:de:de:d#e:fd)d*Z@de:de:de:fd+d,ZAd-d. ZB			d;d/e:de:d0e:ded1eee:e:f  d2eCd$e;fd3d4ZD		d<d/e:de:d0e:d2eCded1eee:e:f  d$e;fd5d6ZE	d=d/e:de:d0e:ded1eee:e:f  f
d7d8ZF	d>d9d:ZGdS )?    N)ThreadPoolExecutor)partial)	CookieJar)Path)DictOptionalUnion)Retry)tqdm)HubApiModelScopeConfig)API_FILE_DOWNLOAD_CHUNK_SIZEAPI_FILE_DOWNLOAD_RETRY_TIMESAPI_FILE_DOWNLOAD_TIMEOUT	FILE_HASHMODELSCOPE_DOWNLOAD_PARALLELS)MODELSCOPE_PARALLEL_DOWNLOAD_THRESHOLD_MBTEMPORARY_FOLDER_NAME)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORT)get_dataset_cache_rootget_model_cache_root)
get_logger   )FileDownloadErrorInvalidParameterNotExistError)ModelFileSystemCache)file_integrity_validationget_endpointmodel_id_to_group_owner_nameFmodel_id	file_pathrevision	cache_dir
user_agentlocal_files_onlycookies	local_dirreturnc                 C   s   t | |t||||||d	S )aF  Download from a given URL and cache it if it's not already present in the local cache.

    Given a URL, this function looks for the corresponding file in the local
    cache. If it's not there, download it. Then return the path to the cached
    file.

    Args:
        model_id (str): The model to whom the file to be downloaded belongs.
        file_path(str): Path of the file to be downloaded, relative to the root of model repo.
        revision(str, optional): revision of the model file to be downloaded.
            Can be any of a branch, tag or commit hash.
        cache_dir (str, Path, optional): Path to the folder where cached files are stored.
        user_agent (dict, str, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional):  If `True`, avoid downloading the file and return the path to the
            local cached file if it exists. if `False`, download the file anyway even it exists.
        cookies (CookieJar, optional): The cookie of download request.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.

    Returns:
        string: string of local file or if networking is off, last version of
        file cached on disk.

    Raises:
        NotExistError: The file is not exist.
        ValueError: The request parameter error.

    Note:
        Raises the following errors:

            - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
            if `use_auth_token=True` and the token cannot be found.
            - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
            if ETag cannot be determined.
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            if some parameter value is invalid
    	repo_typer&   r'   r(   r)   r*   r+   N)_repo_file_downloadr   )r$   r%   r&   r'   r(   r)   r*   r+    r0   `/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/modelscope/hub/file_download.pymodel_file_download)      .r2   
dataset_idc                 C   s   t | |t||||||d	S )a  Download raw files of a dataset.
    Downloads all files at the specified revision. This
    is useful when you want all files from a dataset, because you don't know which
    ones you will need a priori. All files are nested inside a folder in order
    to keep their actual filename relative to that folder.

    An alternative would be to just clone a dataset but this would require that the
    user always has git and git-lfs installed, and properly configured.

    Args:
        dataset_id (str): A user or an organization name and a dataset name separated by a `/`.
        file_path (str): The relative path of the file to download.
        revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
            commit hash. NOTE: currently only branch and tag name is supported
        cache_dir (str, Path, optional): Path to the folder where cached files are stored, dataset file will
            be save as cache_dir/dataset_id/THE_DATASET_FILES.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.
        user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        cookies (CookieJar, optional): The cookie of the request, default None.
    Raises:
        ValueError: the value details.

    Returns:
        str: Local folder path (string) of repo snapshot

    Note:
        Raises the following errors:
        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
        if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
        ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
        if some parameter value is invalid
    r-   N)r/   r   )r4   r%   r&   r'   r+   r(   r)   r*   r0   r0   r1   dataset_file_downloadc   r3   r5   )r.   r&   r'   r(   r)   r*   r+   disable_tqdmrepo_idr.   r6   c             	   C   s  |st }|tvrtd|tf t| |||d\}
}|r0||}|d ur,td |S tdt }dt	j
|di}|d u rCt	 }g }d }|t kr|j| ||d}|j| |d|d u r^d	n|d
}|D ]}|d dkrmqd|d |kr||r|d }td| d ||  S |} qqd|tkrt| \}}|st}d}d}	 |j|||dd||d}d|v r|d dkstd| |d |d f  d S |d d }d	}|D ]0}|d dkrq|d |kr||r|d }td| d ||  S |}d} qqt||k s|rq|d7 }q|d u r$td|| f |t kr0t| ||}n|tkrA|j|d |||d}ntd| t|||
|||S )Nz'Invalid repo type: %s, only support: %s)r+   r'   r.   z>File exists in local cache, but we're not sure it's up to datezCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z
user-agent)r(   )r&   r*   TF)r$   r&   	recursiveZuse_cookiesTypetreer   NamezFile z8 already in cache with identical hash, skip downloading!r   d   /)dataset_name	namespacer&   	root_pathr8   page_number	page_sizeZCode   z=Get dataset: %s file list failed, request_id: %s, message: %sZ	RequestIdMessageZDataZFilesz"The file path: %s not exist in: %s)	file_namer>   r?   r&   zInvalid repo type )r   r   r   $create_temporary_directory_and_cacheZget_file_by_pathloggerwarning
ValueErrorr   r   Zget_user_agentZget_cookiesZget_valid_revisionZget_model_filesexistsdebugZget_file_by_infor   r#   r   Zlist_repo_treeprintlenr   get_file_download_urlZget_dataset_file_urldownload_file)r7   r%   r.   r&   r'   r(   r)   r*   r+   r6   temporary_cache_dircacheZcached_file_pathZ_apiheadersZ
repo_filesZfile_to_download_metaZ	repo_filerE   group_or_ownernamerA   rB   Zfiles_list_treeZis_existZurl_to_downloadr0   r0   r1   r/      s   









%

r/   c                 C   s   |  tjjr| tjj} tj| }tj| }|dkr d S | ds,tj|d}t|\}}|	dd}tj| ||}tj|||}tj
|rstj
|sutd| d|  z	t|| W d S  tyr   Y d S w d S d S )NZdatasetsZhub.___zLegacy cache dir exists: z
, move to )endswithospathsepstripdirnamebasenamejoinr#   replacerJ   rG   infoshutilmove	Exception)r'   r$   Zlegacy_cache_root	base_namerS   rT   rP   Zlegacy_cache_dirr0   r0   r1   !move_legacy_cache_to_standard_dir"  s4   


re   c           	      C   s   |t krt }n|tkrt }ntd| t| \}}|d ur-tj|t	}t
|}n)|d u r8|}t||  t|trAt|}tj|t	||}|dd}t
|||}tj|dd ||fS )Nz7repo_type only support model and dataset, but now is : rU   rV   Texist_ok)r   r   r   r   rI   r#   rX   rY   r^   r   r    re   
isinstancer   strr_   makedirs)	r$   r+   r'   r.   Zdefault_cache_rootrS   rT   rP   rQ   r0   r0   r1   rF   E  s.   



rF   c                 C   s0   t j|}t j|}d}|jt | ||dS )a  Format file download url according to `model_id`, `revision` and `file_path`.
    e.g., Given `model_id=john/bert`, `revision=master`, `file_path=README.md`,
    the resulted download url is: https://modelscope.cn/api/v1/models/john/bert/repo?Revision=master&FilePath=README.md

    Args:
        model_id (str): The model_id.
        file_path (str): File path
        revision (str): File revision.

    Returns:
        str: The file url.
    zQ{endpoint}/api/v1/models/{model_id}/repo?Revision={revision}&FilePath={file_path})Zendpointr$   r&   r%   N)urllibparse
quote_plusformatr"   )r$   r%   r&   Zdownload_url_templater0   r0   r1   rN   e  s   rN   c              
   C   s  | \}}}}}}}}|d u ri nt |}	tt j|	d< ttddgd}
|d||f  }	 zwd}tj	
|rYt|d}|dtj}|| W d    n1 sTw   Y  || }||krdW d S d	||f |	d
< t|d+}tj|d|	|td}|jtdD ]}|r|| |t| qW d    W d S 1 sw   Y  W d S  ty } z|
jd||d}
td||f  |
  W Y d }~nd }~ww q/)NX-Request-IDr   GETtotalbackoff_factorallowed_methods_%s_%sTr   rbbytes=%s-%sRangeab+streamrR   r*   timeout
chunk_sizeerrorz-Downloading: %s failed, reason: %s will retry)copydeepcopyri   uuiduuid4hexr	   r   rX   rY   rJ   openseekioSEEK_ENDupdaterequestsgetr   iter_contentr   writerM   rc   	incrementrG   rH   sleep)paramsZmodel_file_pathprogressstartendurlrE   r*   rR   get_headersretrypart_file_namepartial_lengthfZdownload_startrchunker0   r0   r1   download_part_with_retry}  sh   


r   r   rE   rR   	file_sizec                 C   s  t ddd|dd| d d|d}d}g }	tj||}
tjtj|
dd	 tt|| D ]}|| }|d
 | d
 }|	|
|||| |||f q/|d
 |k rb|	|
||d
 |d
 | |||f t	dkrht	nd}t
|dd}t|t|	 W d    n1 sw   Y  W d    n1 sw   Y  t }ttj||dN}|	D ]@}|d d|d |d f  }t|d}	 |dt }|sq|| || qW d    n1 sw   Y  t| qW d    | S 1 sw   Y  | S )NBT   r   Downloading []unit
unit_scaleunit_divisorrr   initialdescZleavedisablei   
rf   r      download)max_workersthread_name_prefixwbru         rv      )r
   rX   rY   r^   rj   r\   rangeintappendr   r   listmapr   hashlibsha256r   readr   r   r   remove	hexdigest)r   r+   rE   r*   rR   r   r6   r   Z	PART_SIZEtasksr%   idxr   r   Z	parallelsexecutorhash_sha256Zoutput_filetaskr   Z	part_filer   r0   r0   r1   parallel_download  sn   

	


r   c              
   C   s|  |du ri nt |}tt j|d< tj||}tj	tj
|dd td| | d}	t }
ttddgd	}	 ztd
dd|dkrG|nddd| d d|d}|dkr|t|d |d W d   n1 smw   Y  	 W d   W q5d}tj|rd}	t|d}|dtj}|| W d   n1 sw   Y  ||kr	 W d   W q5d||d f |d< t|d4}tj| d||td}|  |jtdD ]}|r|t| || |	s|
| qW d   n1 sw   Y  W d   n	1 sw   Y  W q5 t y3 } zd}	|j!d| |d}|"  W Y d}~nd}~ww q<|	r:dS |
# S )a  Download remote file, will retry 5 times before giving up on errors.

    Args:
        url(str):
            actual download url of the file
        local_dir(str):
            local directory where the downloaded file stores
        file_name(str):
            name of the file stored in `local_dir`
        file_size(int):
            The file size.
        cookies(CookieJar):
            cookies used to authentication the user, which is used for downloading private repos
        headers(Dict[str, str], optional):
            http headers to carry necessary info when requesting the remote file
        disable_tqdm(bool, optional): Disable the progress bar with tqdm.

    Raises:
        FileDownloadError: File download failed.

    Nro   Trf   downloading %s to %sFr   rp   rq   r   r   r   r   r   r   zw+rv   rw   rx   ry   rz   r}   r   )$r   r   ri   r   r   r   rX   rY   r^   rj   r\   rG   rK   r   r   r	   r   r
   r   r   rJ   r   r   r   r   r   r   raise_for_statusr   r   rM   r   rc   r   r   r   )r   r+   rE   r   r*   rR   r6   r   Ztemp_file_pathZ	has_retryr   r   r   r   r   r   r   r   r0   r0   r1   http_get_model_file  s   
	



/7r   c                 C   s  d}t tjd|dd}|du ri nt|}tt j|d< | }t	
d| |j ttdd	gd
}		 zS| }
d|
 |d< tj| d||td}|  |jd}|dur[t|nd}tddd||
d| d d}|jtdD ]}|r|t| || qq|  W q ty } z|	jd	| |d}	|	  W Y d}~nd}~ww q4W d   n1 sw   Y  t	
d| | tj !|j}||krt"|j d|||f }t	#| t$|t%|jtj &|| dS )aj  Download remote file, will retry 5 times before giving up on errors.

    Args:
        url(str):
            actual download url of the file
        local_dir(str):
            local directory where the downloaded file stores
        file_name(str):
            name of the file stored in `local_dir`
        cookies(CookieJar):
            cookies used to authentication the user, which is used for downloading private repos
        headers(Dict[str, str], optional):
            http headers to carry necessary info when requesting the remote file

    Raises:
        FileDownloadError: File download failed.

    r   F)modedirdeleteNro   r   r   rp   rq   Tz	bytes=%d-rx   rz   zContent-Lengthr   r   r   r   )r   r   r   rr   r   r   r}   r   zstoring %s in cache at %sz}File %s download incomplete, content_length: %s but the                     file downloaded length: %s, please download again)'r   tempfileNamedTemporaryFiler   r   ri   r   r   r   rG   rK   rT   r	   r   tellr   r   r   r   rR   r   r
   r   r   r   rM   r   closerc   r   r   rX   rY   getsizer   r   r   r_   r^   )r   r+   rE   r*   rR   rr   Ztemp_file_managerr   	temp_filer   Zdownloaded_sizer   content_lengthr   r   r   Zdownloaded_lengthmsgr0   r0   r1   http_get_fileB  s   



(
r   c           
   	   C   s   t d d |d k r&tdkr&t| ||d ||d u rd n| |d |d}nt| ||d |d |||d}tj||d }t|v r]|t }	|d urX||	krWt	d t
||	 nt
||	 |||S )Ni  Sizer   r   )rR   r*   r   r6   )r   rR   r*   r6   zLMismatched real-time digest found, falling back to lump-sum hash computation)r   r   r   get_dictr   rX   rY   r^   r   rL   r!   Zput_file)
r   Z	file_metarP   rQ   rR   r*   r6   Zfile_digestr   Zexpected_hashr0   r0   r1   rO     sF   	


rO   )NNF)NF)N)F)Hr   r   r   rX   ra   r   rk   r   concurrent.futuresr   	functoolsr   http.cookiejarr   pathlibr   typingr   r   r   r   Zrequests.adaptersr	   Z	tqdm.autor
   Zmodelscope.hub.apir   r   Zmodelscope.hub.constantsr   r   r   r   r   r   r   Zmodelscope.utils.constantr   r   r   r   r   Zmodelscope.utils.file_utilsr   r   Zmodelscope.utils.loggerr   errorsr   r   r   Zutils.cachingr    Zutils.utilsr!   r"   r#   rG   ri   boolr2   r5   r/   re   rF   rN   r   r   r   r   r   rO   r0   r0   r0   r1   <module>   sR  $	
=	
>	

 $
 /
=
i
Y