
    OgB                         S SK Jr  S SK Jr  S SK Jr  S SK Jr  S SKrS SKrS SKrS SK	r	S SK
r
 S SKJr  1 SkrSS jrSS	 jrS
 rS rS rSS jrS rSS jrg! \ a	    S SKJr   N3f = f)    )absolute_import)division)print_function)unicode_literalsN)urlopen>   afamanarasazbabebgbhbnbobrbscacecocscvcydadedveleneoeseteufafifrfygagdglgugvhehihrhthuhyiaidioisitjajvkakkkmknkokukylalbliltlvmgmkmlmnmrmsmtmynenlnnnoocorospaplpsptqurmrorusascsdshsiskslsosqsrsusvswtatetgthtktltrttugukuruzvivowayiyozhalsarzastazbbarbclbpycebckbdiqemlfrrgomhifhsbilolmomaimhrminmrjmwlmyvmznnahnapndsnewnsopampflpmspnbsahscnscovecvlswarxmfzeac                     SnSnSn[        X5       H-  u  paU H  nXq;   d  M
  US-  nM     US-  nU[        U5      -  nM/     X2U-  -  X5-  4$ )z;
Return precision and recall modeled after fasttext's test
g        r      )ziplen)predictionslabelsk	precision	nexamplesnlabels
predictionps           W/Users/admin/workspace/ai/Jarvis/env/lib/python3.13/site-packages/fasttext/util/util.pytestr   7   sm     IIG!+6
A{Q	  	Q	3v; 7 Y')<==    c                    Uc  [         R                  " XUS9nO[         R                  " XUS9  [        U5      S-
  n[         R                  " X45      U   nXR;   a%  US-  n[         R                  " X45      U   nXR;   a  M%  U$ )a  
query is a 1d numpy array corresponding to the vector to which you want to
find the closest vector
vectors is a 2d numpy array corresponding to the vectors you want to consider
ban_set is a set of indicies within vectors you want to ignore for nearest match
cossims is a 1d numpy array of size len(vectors), which can be passed for efficiency

returns the index of the closest match to query within vectors

)outr   )npmatmulr   argpartition)queryvectorsban_setcossimsrankresult_is         r   find_nearest_neighborr   G   sy     ))G8
		'g.w<!Dw-d3H

	??71$7 
 Or   c                    Uc  SnU SU nXDR                  S[        R                  S9-
  n[        R                  " [        R                  " UR
                  U5      UR                  S   S-
  [        R                  S9n[        R                  R                  U5      u  pgUSS2SU24   n[        R                  " X5      nX4$ )u   
Reduces the dimension of a (m × n)   matrix `X_orig` to
                      to a (m × dim) matrix `X_reduced`
It uses only the first 100000 rows of `X_orig` to do the mapping.
Matrix types are all `np.float32` in order to avoid unncessary copies.
Ni r   )axisdtyper   )r   )	meanr   float32divider   Tshapelinalgeig)	X_origdimeigvmapping_sizeXC_U	X_reduceds	            r   _reduce_matrixr   ^   s     |=L!ARZZ00IIbiiQ'arzzJyy}}QDSDz		&'Ir   c                     [        U R                  5       US5      u  p#[        U R                  5       X5      u  pEU R                  X$5        U $ )z
ft_model is an instance of `_FastText` class
This function computes the PCA of the input and the output matrices
and sets the reduced ones.
N)r   get_input_matrixget_output_matrixset_matrices)ft_model
target_diminp_reducedprojout_reducedr   s         r   reduce_modelr   r   sP     '!!#Z7K#""$j8NK +3Or   c                 .   [        U 5      U-  nSn[        X#-  5      n[        US-  S5      n[        R                  R                  SU-  5        [        R                  R                  SU-  5        [        R                  R                  S5        [        R                  R                  SX4-
  -  5        [        R                  R                  S5        [        R                  R                  5         X:  a   [        R                  R                  S	5        g g )
N2   d      z (%0.2f%%) [=> z]
)floatintroundsysstdoutwriteflush)downloaded_bytes
total_sizepercentbar_sizer   s        r   _print_progressr      s    $%
2GH
g 
!CGcM1%GJJ^g-.JJS3YJJSJJSHN+,JJUJJ%

 &r   c                 (   [        SU -  5        [        U 5      n[        US5      (       a)  [        UR	                  S5      R                  5       5      nO6[        UR                  5       R	                  S5      R                  5       5      nSnUS-   n[        US5       n UR                  U5      nU[        U5      -  nU(       d  OUR                  U5        [        XT5        ME  S S S 5        [        R                  " Xa5        g ! , (       d  f       N%= f)NzDownloading %s	getheaderzContent-Lengthr   z.partwb)printr   hasattrr   r   stripinfoopenreadr   r   r   rU   rename)	urlwrite_file_name
chunk_sizeresponse	file_size
downloadeddownload_file_namefchunks	            r   _download_filer     s    	
S
 !s|Hx%%**+;<BBDE	112BCIIKL	J(72	 $	'1MM*-E#e*$JGGENJ2  
( II 2 
(	's   AD
Dc                     [         R                  R                  U 5      (       a   US:X  a  gUS:X  a  [        S5        gUS:X  a   SU -  n[	        X 5        g)NignoreTstrictz5gzip File exists. Use --overwrite to download anyway.F	overwritez8https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/%s)rU   pathisfiler   r  )gz_file_name	if_existsr   s      r   _download_gz_modelr     sT    	ww~~l## ("IJ+%
D|
SC3%r   c                    U [         ;  a  [        S[        [         5      -  5      eSU -  nSU-  n[        R                  R                  U5      (       a!  US:X  a  U$ US:X  a  [        S5        gUS:X  a   [        XA5      (       aM  [        R                  " US	5       n[        US
5       n[        R                  " XV5        SSS5        SSS5        U$ U$ ! , (       d  f       N= f! , (       d  f       U$ = f)zr
Download pre-trained common-crawl vectors from fastText's website
https://fasttext.cc/docs/en/crawl-vectors.html
z'Invalid lang id. Please select among %szcc.%s.300.binz%s.gzr  r	  z0File exists. Use --overwrite to download anyway.Nr
  rbr   )valid_lang_ids	ExceptionreprrU   r  r  r   r  gzipr   shutilcopyfileobj)lang_idr  	dimension	file_namer  r  f_outs          r   download_modelr     s    
 n$A^,- . 	.  ')IY&L	ww~~i   ("DE+%,22YY|T*ai&%""1, ' + 9 '& +* s$   C%)C C%
C"	C%%
C4)r   )N)i    )r	  N)
__future__r   r   r   r   numpyr   r   r  rU   r  urllib.requestr   ImportErrorurllib2r  r   r   r   r   r   r  r  r   r   r   <module>r#     sn   " '  % '  
  	  &
9*> .(  3* s    s   A A('A(