o
    "ѹgG                     @   s$  d dl mZmZmZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZmZmZ ddlmZmZ dd	lmZ ed
ZedZd dlmZmZmZ d dlm
Z
 d dlmZmZmZ dd Z dVddZ!dd Z"dd Z#dd Z$dd Z%dd Z&ej'(ddd gd!d" Z)d#d$ Z*G d%d& d&e+Z,dWd'd(Z-d)d* Z.d+d, Z/ej'(d-e0e1dd.d/ Z2d0d1 Z3d2d3 Z4d4d5 Z5d6d7 Z6d8d9 Z7d:d; Z8d<d= Z9d>d? Z:d@dA Z;dBdC Z<dDdE Z=ej'(dFdGdHgej'j>ej?dIkoej?dJkdKdLdMdN Z@dOdP ZAej'(dQdRdSgdTdU ZBdS )X    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_config)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributeddask)ClientLocalCluster
get_client)time)clusterinccleanupc                  O      d S N )argskwargsr   r   Y/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/joblib/test/test_dask.pynoop   s   r   皙?c                 C   s   t | | r
tdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s   r$   c                    sF   | dd }i }| D ]\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)Zdask_workerr   r   r   <lambda>$   s    zcount_events.<locals>.<lambda>c                    s   g | ]
}|d   kr|qS    r   ).0event
event_namer   r   
<listcomp>'   s    

z count_events.<locals>.<listcomp>)runitemslenlist)r,   clientZworker_eventsZevent_countsweventsr   r+   r   count_events#   s
    r5   c                 C   sD  t  \}\}}t|d | dp}tdd[ t dd tdD }|dd	 tdD ks0J tt t d
d tdD  W d    n1 sLw   Y  t dd tdD }|dd	 tdD ksjJ W d    n1 stw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Naddressloopr   backendc                 s       | ]	}t t|V  qd S r   r   r   r)   ir   r   r   	<genexpr>0       ztest_simple.<locals>.<genexpr>
   c                 S      g | ]}t |qS r   r   r=   r   r   r   r-   1       ztest_simple.<locals>.<listcomp>c                 s   s     | ]}t t|d kV  qdS )   N)r   r$   r=   r   r   r   r?   4   s    c                 s   r;   r   r<   r=   r   r   r   r?   7   r@   c                 S   rB   r   rC   r=   r   r   r   r-   8   rD   )r   r   r   r
   rangepytestraisesr!   r8   sabr2   seqr   r   r   test_simple,   s$   

"rN   c                 C   s,  t jtju sJ t \}\}}t|d | d\}tddG t 4}|j}t|t s-J |j	|u s4J |j
dks;J |dd ttdD  |j
d	ksOJ W d    n1 sYw   Y  W d    n1 shw   Y  W d    n1 sww   Y  W d    d S W d    d S 1 sw   Y  d S )
Nr6   r7   r   r9   r(   c                 s   s    | ]
}t d d  V  qdS )c                   S   r   r   r   r   r   r   r   r&   M   s    z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r)   _r   r   r   r?   L   
    
z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @rA   )r   Zcompute_batch_sizer   r   r   r   r
   _backend
isinstanceparallelZ_effective_batch_sizerF   int)r8   rJ   rK   rL   r2   rV   r:   r   r   r   #test_dask_backend_uses_autobatching;   s.   
"rX   c                   C   s   t  S r   r   r   r   r   r   random2S   s   rY   c              
   C   s   t  V\}\}}t|d | d1}tdd t dd tdD \}}||ks+J W d    n1 s5w   Y  W d    n1 sDw   Y  W d    d S W d    d S 1 s\w   Y  d S )Nr6   r7   r   r9   c                 s       | ]}t t V  qd S r   )r   rY   r=   r   r   r   r?   [   s    z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   rF   )r8   rJ   rK   rL   r2   xyr   r   r    test_dont_assume_function_purityW   s   "r^   mixedTFc              
   C   s4  ddl m} |sdd tdD }d}ndd tdD }d}t|||ks)J t d\}\}}t|d	 | d
?}tdd tddd|}	W d    n1 sRw   Y  dd }
|dd}|	|
}t
dd |D sqJ W d    n1 s{w   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r<   r=   r   r   r   r-   c       z&test_dask_funcname.<locals>.<listcomp>   Zbatch_of_inc_4_callsc                 S   s,   g | ]}|d  rt t|nt t|qS )r	   )r   absr   r=   r   r   r   r-   f   s     Zmixed_batch_of_inc_4_callsr6   r7   r   r9   r	   all)Z
batch_sizeZpre_dispatchc                 S   s
   t | jS r   )r1   Ztransition_log)Zdask_schedulerr   r   r   fr      
ztest_dask_funcname.<locals>.f42c                 s   s    | ]	}d |d v V  qdS )Zbatch_of_incr   Nr   )r)   tupr   r   r   r?   v   r@   z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskr`   rF   reprr   r   r   r
   replaceZrun_on_schedulerrd   )r8   r_   r`   tasksZ
batch_reprrJ   rK   rL   r2   rQ   re   r%   r   r   r   test_dask_funcname_   s,   
"rm   c               	      s  dd t dD } tdtd d fdd	tdd	d
}t|}ztdd t fdd| D }W d    n1 sDw   Y  | dd t dD ksVJ t	d|}t
| dkseJ tdd |D spJ tdd t  fdd| D }W d    n1 sw   Y  t	d|}t
| dksJ tdd |D sJ W |jdd |jdd d S |jdd |jdd w )Nc                 S      g | ]}g qS r   r   rP   r   r   r   r-          z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d urj |  | t j | S r   )ZtestingZassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s   zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr(   r	   )	n_workersthreads_per_workerr   r9   c                 3   s    | ]	}t  |V  qd S r   rO   r)   rt   )rx   r   r   r?          
z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   rn   r   r   rP   r   r   r   r-      ro   receive-from-scatterr   c                 S      g | ]}t |d kqS r'   r0   r)   rr   r   r   r-      ra   c                 3   s     | ]}t | d V  qdS ))ru   NrO   r{   )rv   rx   r   r   r?      s    
c                 S   r~   r'   r   r   r   r   r   r-      ra      timeoutr   )rF   rG   importorskipZarangerW   r   r   r   r
   r5   sumvaluesrd   close)listsr   r2   rescountsr   )rv   rx   rw   r   'test_no_undesired_distributed_cache_hity   s8   




r   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )r\   count)selfr\   r   r   r   __init__   s   
zCountSerialized.__init__c                 C   s   | j t|d| S )Nr\   )r\   getattr)r   otherr   r   r   __add__   s   zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr(   )r   r   r\   )r   r   r   r   
__reduce__   s   zCountSerialized.__reduce__N)__name__
__module____qualname__r   r   __radd__r   r   r   r   r   r      s
    r   c                 C   s   | | | | | S r   r   )rK   rL   cder   r   r   add5   s   r   c                 C   s  t d}t d}t d}t \}\}}t|d | d~}td||gd: tt}||||dd	d
||||d	dd
|||||d	d
||||||d
g}	dd |	D }
t |	}W d    n1 sbw   Y  tt	 td| dd W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  ||
ksJ |j
dksJ |j
dksJ |j
dv sJ d S )Nr(   r	   rE   r6   r7   r   )r:   scatterrb      )r   r   c                 S   s    g | ]\}}}||i |qS r   r   )r)   funcr   r   r   r   r   r-      s    z'test_manual_scatter.<locals>.<listcomp>)r:   r8   r   )rb      )r   r   r   r   r   r   r
   rG   rH   	TypeErrorr   )r8   r\   r]   zrJ   rK   rL   r2   re   rl   expectedresultsr   r   r   test_manual_scatter   s>   r   c           	   
      s  t d}|jtd|jd |jtd|jd} gd |gd  }t Z\}\}}t|d | d>}tdd t d	d
 t	|D  W d    n1 sPw   Y  t
d|}||d  ||d   dksjJ W d    n1 stw   Y  W d    n1 sw   Y  t i\}\}}t|d | dD}tdd t  fdd
tdD  W d    n1 sw   Y  t
d|}||d  dksJ ||d  dksJ W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nrq   rS   )ZdtyperE   r6   r7   r   r9   c                 s   s(    | ]\}}t t||||d V  qdS ))optNr   r   )r)   r>   ru   r   r   r   r?      s    z$test_auto_scatter.<locals>.<genexpr>r}   r	   c                 3   s&    | ]}t t d d |V  qd S )NrE   r   r=   Zdata1r   r   r?      s   $ r   r   )rG   r   onesrW   Zuint8r   r   r   r
   	enumerater5   rF   )	loop_in_threadrw   Zdata2Zdata_to_processrJ   rK   rL   r2   r   r   r   r   test_auto_scatter   s8   



"
"r   retry_noc              
      s   t dd d}fdd fddt V\}\}}t|d | d1}td	d
 dt fddt|D }W d    n1 sKw   Y  W d    n1 sZw   Y  W d    d S W d    d S 1 srw   Y  d S )Nrq   rA   c                    s
     | S r   )r   )r\   r>   j)rw   r   r   my_sum  rf   z#test_nested_scatter.<locals>.my_sumc                    s^   t  }tdd t  fddtD }W d    t|S 1 s&w   Y  t|S )Nr   r9   c                 3   s(    | ]}t  |d  |V  qd S r   rO   )r)   r   )arrayr>   r   r   r   r?     s    
zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rF   r   )r   r>   r2   r   )NUM_INNER_TASKSr   )r   r>   r   outer_function_joblib  s   

z2test_nested_scatter.<locals>.outer_function_joblibr6   r7   r   r9   i'  c                 3   s&    | ]}t  |d  |V  qd S r   rO   r=   )my_arrayr   r   r   r?      s    
z&test_nested_scatter.<locals>.<genexpr>)rG   r   r   r   r   r   r
   rF   )r8   r   ZNUM_OUTER_TASKSrJ   rK   rL   rQ   r   )r   r   r   rw   r   r   test_nested_scatter  s$   
	

"r   c              
      s|  dd  t  \}\}}t|d | d<}tdd' tdd fd	d
tdD }|D ]}tt|dks9J q-W d    n1 sDw   Y  W d    n1 sSw   Y  t|d | d<}tdd' tdd fdd
tdD }|D ]}tt|dksJ qyW d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s       | ]	}t tj V  qd S r   r   osgetpidrP   r   r   r   r?   (  r@   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   r   r   r   rP   r   r   r   r?   )  r@   )setr
   rF   )Zpidsr   r   r   get_nested_pids'  s    $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr6   r7   r   r9   r	   r   c                 3       | ]}t   V  qd S r   rO   rP   r   r   r   r?   /  
    
z6test_nested_backend_context_manager.<locals>.<genexpr>rA   c                 3   r   r   rO   rP   r   r   r   r?   9  r   )r   r   r   r
   rF   r0   r   )r   rJ   rK   rL   r2   Z
pid_groupsZ	pid_groupr   r   r   #test_nested_backend_context_manager&  s6   
"r   c           	         s6  dd   fddt  \}\}}t|d | d_}tddJ t $} |d	ks-J |jd
ks4J |fddtdD }W d    n1 sKw   Y  |D ]\}}|d	ks\J |d
ksbJ qRW d    n1 smw   Y  W d    n1 s|w   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                 S   s
   | j jjS r   )rT   	__class__r   pr   r   r   _backend_typeE  rf   zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t  }  | | jfW  d    S 1 sw   Y  d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsH  s   $zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr6   r7   r   r9   r   c                 3   r   r   rO   rP   )r   r   r   r?   R  r   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   r   rF   )	r8   rJ   rK   rL   r2   r   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobsA  s,   
"r   c              	   C   sn   t t}tdd W d    n1 sw   Y  W d    n1 s%w   Y  dt|j v s5J d S )Nr   r9   zcreate a dask client)rG   rH   r!   r   strvaluelower)r8   infor   r   r   test_errors[  s   r   c              
   C   s4  t  \}\}}t|d | dh}tdd# tdddd td	D }t|d
 d
 d
 ts2J W d    n1 s<w   Y  tdd# tdddd td	D }t|d
 d
 d
 tsbJ W d    n1 slw   Y  W d    n1 s{w   Y  W d    d S W d    d S 1 sw   Y  d S )Nr6   r7   r   r9   r	   r   c                 s   s    | ]
}t td dV  qd S )NZnested_requirer   outerrP   r   r   r   r?   h  s    
z.test_correct_nested_backend.<locals>.<genexpr>r(   r   c                 s   s    | ]
}t td dV  qdS )Z	sharedmemr   Nr   rP   r   r   r   r?   n  rR   )r   r   r   r
   rF   rU   r   r   )r8   rJ   rK   rL   r2   resultr   r   r   test_correct_nested_backendc  s&   "r   c                    s"   t ddd fddtdD S )Nr	   threads)r   Zpreferc                 3   s    | ]	}t t V  qd S r   )r   middlerP   r   r   r   r?   u  r|   zouter.<locals>.<genexpr>r(   r
   rF   r   r   r   r   r   t  s   r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   rZ   r   )r   innerrP   r   r   r   r?   {  s    

zmiddle.<locals>.<genexpr>r(   r   )r   r   r   r   r   z  s   r   c                   C   s   t  jS r   )r
   rT   r   r   r   r   r     s   r   c              	   C   s   t | ddd4 tdd tdddd	 td
D  W d    n1 s&w   Y  W d    d S W d    d S 1 s>w   Y  d S )NFT)r8   	processesZset_as_defaultr   r9   rb   r   c                 s   r;   r   )r   idr=   r   r   r   r?     r@   z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rF   r7   r   r   r   test_secede_with_no_processes  s   "r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r6   )rQ   r   r   r   r   _worker_address  s   r   c              
   C   s0  t  \}\}}t|d | df}td|d d t dd tdD }||d gd ks1J W d    n1 s;w   Y  td|d d t dd tdD }||d gd ks`J W d    n1 sjw   Y  W d    n1 syw   Y  W d    d S W d    d S 1 sw   Y  d S )	Nr6   r7   r   )r:   workersc                 s   r;   r   r   r   r=   r   r   r   r?     r|   z-test_dask_backend_keywords.<locals>.<genexpr>rA   c                 s   r;   r   r   r=   r   r   r   r?     r|   r[   rI   r   r   r   test_dask_backend_keywords  s&   

"r   c              	   C   s   t d| dI}tdd t dd tdD  W d    n1 s#w   Y  t }|jjjrBtd t |d	 k s=J |jjjs0|j	rGJ W d    d S 1 sRw   Y  d S )
NF)r   r8   r   r9   c                 s   r;   r   r<   r=   r   r   r   r?     r@   z/test_scheduler_tasks_cleanup.<locals>.<genexpr>rA   g{Gz?r   )
r   r   r
   rF   r   r   Z	schedulerrl   r   futures)r8   r2   startr   r   r   test_scheduler_tasks_cleanup  s   

"r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc                 C   s   t dddd}t|}| dkr|jddd n	| dkr |d z7tdd	 t d
d tdD  W d    n1 s=w   Y  W |  |  d S W |  |  d S |  |  w )Nr   Fr	   ry   r   rz   r   )Zminimummaximumr   r   r9   c                 s   r;   r   r<   r=   r   r   r   r?     r@   z(test_wait_for_workers.<locals>.<genexpr>rA   )r   r   ZadaptZscaler   r
   rF   r   )r   r   r2   r   r   r   test_wait_for_workers  s"   

r   c               	   C   sB  t dddd} t| }ztddd- d}tjt|d	 t d
d tdD  W d    n1 s3w   Y  W d    n1 sBw   Y  tddd- d}tjt|d	 t dd tdD  W d    n1 snw   Y  W d    n1 s}w   Y  W |	  | 	  d S W |	  | 	  d S |	  | 	  w )Nr   Fr	   r   r   g?)r:   Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   r;   r   r<   r=   r   r   r   r?     r@   z0test_wait_for_workers_timeout.<locals>.<genexpr>rA   z+DaskDistributedBackend has no active workerc                 s   r;   r   r<   r=   r   r   r   r?     r@   )
r   r   r   rG   rH   TimeoutErrorr
   rF   RuntimeErrorr   )r   r2   msgr   r   r   test_wait_for_workers_timeout  s0   
r   r:   Zlokymultiprocessingc              	      s   t dd}t|}z: fdd}||}| }t|dks!J |d j}t|ts-J dt|v s5J W |j	dd	 |j	dd	 d S |j	dd	 |j	dd	 w )
Nr	   )ry   c                     sP   t jdd} td ddd tdD  W d    | S 1 s!w   Y  | S )NTrecordr	   )r   r:   c                 s   r;   r   r<   r=   r   r   r   r?     r|   zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>rA   )warningscatch_warningsr
   rF   r   r9   r   r   func_using_joblib_parallel  s   

zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr(   r   zdistributed.worker.daemonr   r   )
r   r   submitr   r0   messagerU   UserWarningr   r   )r:   r   r2   r   futr   warningr   r9   r   /test_joblib_warning_inside_dask_daemonic_worker  s   


r   )r    )r   r   )C
__future__r   r   r   r   r   rG   r   uuidr   r   r    r
   r   r   rV   r   r   Z_daskr   r   r   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   r   r$   r5   rN   rX   rY   r^   markZparametrizerm   r   objectr   r   r   r   r1   rF   r   r   r   r   r   r   r   r   r   r   r   r   Zskipif__version__r   r   r   r   r   r   r   <module>   sf    


	
9
%
