o "ѹg!@sdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddlm Z ddl mZddlmZddlmZddlZddlZdd lmZdd lmZmZdd lmZdd lmZmZdd lmZddlmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%edurddl&m'Z'ddl(m)Z)zddl*Z*Wn e+ydZ*Ynwzddl,m-Z-Wn e+ydZ-Ynwzddl.Z.Wn e+ydZ.Ynwddl/m0Z0ddl/m1Z1ddl/m2Z2ddl/m3Z3ddl/m4Z4ddl5m6Z6m7Z7ddl5m8Z8ddl5m9Z9ddl5m:Z:ddl5m;Z;mZ>e=?Z@e@AdddgeBe=CZDeDd d!e=D7ZDedur8gZEndd"gZEeEd#gZFeGed$rOeDHeId%e=e>ZJd&d'ZKd(d)ZLd*d+ZMGd,d-d-eNZOdPd/d0ZPd1d2ZQdQd3d4ZRd5d6ZSd7d8ZTd9d:ZUd;d<ZVe!d=e8e9gejWj!d>d?d@e;d@dAfdBggdCdDedEdFZXe!dGeDe!dHgdIe!dJgdKdLdMZYe!dGeDdNdOZZdPdQZ[ee!dRgdSdTdUZ\ee!dGgdVdWdXZ]dYdZZ^e!d[e=e!d\e=d]d^Z_d_d`Z`edadbZadcddZbe!dHgdedfdgZce!dGeFdhdiZdedjdkZeeee!dlgdmdndoZfe!dGeFdpdqZgee!dGeFdrdsZhee!dGeEdtduZiee!dGe=dvdwZjdxdyZke!dGe=e!dzdgd{fd|gd}fgd~dZlee!dGeFddZmddZnee!dGeEddZoddZpddZqddZree!dGeFejWjsddddZtGddde0ZuddZve!dGeDddZwee!dGeFe!dHgdddZxddZyddZze$eduddddZ{ddZ|eFddZ}e}~dd!edDee!dGe}e!d=e9e8gddZGddde0Ze!d=e8e9gddZe!d=e8e9gddZddZddZGdddejj4Ze:deee!dGgde!d=e8e9gddZee!dHgde!dGeFe!d=e8e9gddZddZee!deFe!deFe!d=e8e9gddZee!d=e8e9ge!ddd.gdd„Ze!dgdĢddƄZe!dgdȢddʄZedd̄Zeee!dGeEdd΄ZdZee!dGeEddфZdZdZdZdZee!dGeEedurgnd%ge!deeege!dgdآddڄZdjejejejd܍ZeddބZdjejejejd܍ZeddZddZddZddZeee!dGeEddZddZeeddZddZddZejW!dHgdddZddZddZejW!dHdd|gee!dGee@dhddZejW!dHdd@ge!d=e8e9ge$e.duddddZe!dGeDe!dHgdddZddZddZee!dGe@e!ddd ge!dHgdd d ZeejW!dHdd@ge!ddd ge!d=e8e9ge$e.duddd d Ze!dGe@e!ddd ge!dHgdddZe!dGe@e!ddd ge!dHgdddZe!dGe@e!ddd ge!ddd ge!dHgdddZe!dgde!ddd ge!ddd gddZeee!dGeEddZe!dGedurdd#gngdddZee!dGeEdd Zee!d=e8e9gd!d"Ze!d=e8e9gd#d$Zd%d&ZdRd'd(Zee!dGd"d#ge!d=e8e9gd)d*Zee!d=e8e9ge$e.duddd+d,ZdSd-d.ZejWje!d=e8e9ge!dGedur4d#gnd"d#gd/d0Zd1d2Ze!dGedurRdd"gndge$e-dud3dd4d5Ze!d=e8e9ge$e.duddd6d7Ze!d=e8e9gd8d9ZƐd:d;Zǐd<d=ZȐd>d?Zɐd@dAZeee!dHgdBdCdDZeee!dEgdFe!dHdd@ge!d=e8e9gdGdHZee!dHdd@ge!dIgdJe!d=e8e9gdKdLZee!dHgdMdNdOZdS(Tz Test the parallel module. N)format_exceptionsqrt)sleep) PicklingError) nullcontext TimeoutErrorparallel)dumpload)mp)np with_numpy)with_multiprocessing)IS_PYPY force_gc_pypy) parametrizeraisescheck_subprocess_callskipifwarns)get_reusable_executor)Queue) parallel_sum)SequentialBackend)ThreadingBackend)MultiprocessingBackend)ParallelBackendBase) LokyBackend)Paralleldelayed)parallel_config)parallel_backend)register_parallel_backend)effective_n_jobs cpu_count)rBACKENDSDEFAULT_BACKENDmultiprocessingcCsg|]}t|qS)r)).0Z backend_strr,r,]/Users/admin/.pyenv/versions/3.10.0/lib/python3.10/site-packages/joblib/test/test_parallel.py Msr/loky threading get_contextspawncCst|dt|ddS)N_poolZ_workers)getattrbackendr,r,r. get_workers[sr8cCs||SNr,)xyr,r,r.division_r<cC|dSNr,r:r,r,r.squarecr=rBc@seZdZdZddZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cCdSr9r,)selfabcdr,r,r.__init__jz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rJr,r,r,r.rCgs rCFcCs"|dkr|r tddddt|S)NrFrGrHrI)rC ValueError)r:custom_exceptionr,r,r.exception_raiserns rTcCstdt)Ng?)timerKeyboardInterruptrAr,r,r.interrupt_raiserus rWcCs|d||S)zO A module-level function so that it can be spawn with multiprocessing. r@Nr,)r:r;zr,r,r.fzsrYcCsttdSNr)typer get_active_backendr,r,r,r._active_backend_typesr]cCst||dddtdDS)Nn_jobsr7cs|] }tt|VqdSr9r#rBr-ir,r,r.   z parallel_func..r"range) inner_n_jobsr7r,r,r. parallel_funcsrjcCtdksJdSrZ)r(r,r,r,r.test_cpu_countrlcCrkrZr'r,r,r,r.test_effective_n_jobsrmrocontextzbackend_n_jobs, expected_n_jobs)rfrfr_Nr)z positive-intz negative-intNone)idscCsT|d|dtdd|ksJWdn1swYtdddks(JdS)Nr1rrrrn)rpZbackend_n_jobsZexpected_n_jobsr,r,r.test_effective_n_jobs_Nones rvr7r_)rr@rqverbose)r@ dcCs:ddtdDt|||dddtdDksJdS)NcSg|]}t|qSr,rBr-r:r,r,r.r/z(test_simple_parallel..)r_r7rxcsr`r9rar}r,r,r.rdrez'test_simple_parallel..rhr")r7r_rxr,r,r.test_simple_parallels rcCs|jtdddtjdd}td|ddd td D}|gd ks(JWdn1s2wYd d |D}t|dksFJdS)NnameZ!some_new_name_for_the_main_thread)targetrvalueTrecordr@r^csr`r9rar}r,r,r.rdrez6test_main_thread_renamed_no_warning..rf)rrcSsg|] }dt|jvr|qS)zworker timeout)strmessager-wr,r,r.r/z7test_main_thread_renamed_no_warning..r)setattrr1current_threadwarningscatch_warningsr"rhlen)r7 monkeypatchwarninforesultsr,r,r.#test_main_thread_renamed_no_warnings rcCstjdd}tdt||dWdn1swYdd|D}|rN|rLtdd|D}ttjd d rBt|d knt|d k}|oK|Sd S|rRJdS) NTralways)r7ricSsg|]}|jqSr,)rrr,r,r.r/sz*_assert_warning_nested..css|] }d|jdvVqdS)zbacked parallel loops cannotrN)args)r-Zeachr,r,r.rd  z)_assert_warning_nested..ZnogilFr) rr simplefilterrjallr5sysflagsr)r7riexpectedrZwarnings_are_correctZwarnings_have_the_right_lengthr,r,r._assert_warning_nesteds"  rz%parent_backend,child_backend,expected))r0r+T)r0r0F)r+r+T)r+r0T)r1r+T)r1r0Tcstd|dfddtdDtd|dfddtdD}|dkr;tr3t|s3tjdd t|s9JdSt|sAJdS) Nr@r^c3s"|] }ttdddVqdS)rFr7rirNr#rr-_ child_backendr,r.rd z0test_nested_parallel_warnings..rc3s"|] }ttddVqdS)r@rNrrrrr,r.rdrr1zThis test often fails in PyPy.reason)r"rhranypytestxfailr)parent_backendrrresr,rr.test_nested_parallel_warningss   r)r0r+r1cCs>dg}dd}tj||fd}|||dsJdS)NFcSshtjdd}tddddtdDWdn1swYtt|t|dk|d<dS) NTrr@rrcs|] }ttdVqdS)皙?Nr#rrr,r,r.rdrezPtest_background_thread_parallelism..background_thread..rr)rrr"rhprintr)is_run_parallelrr,r,r.background_thread s  z=test_background_thread_parallelism..background_thread)rrr)r1Threadstartjoin)r7rrtr,r,r."test_background_thread_parallelisms rcCs"td|dddtdDdS)Nr@r^csr){Gz?Nrarr,r,r.rdreznested_loop..rgr6r,r,r. nested_loops rrrcs&td|dfddtdDdS)Nr@r^c3|] }ttVqdSr9)r#rrrr,r.rd!rez#test_nested_loop..rg)rrr,rr.test_nested_loops rcCstr9)rRr6r,r,r.raise_exception%rKrc Cstt5tddd}|ttdttdgWdn1s$wYWddSWddS1s.N)rr"rhfullr,r,rr.test_mutate_input_with_threads1s r)rr@rfcCs6td}dd|Dt|ddd|DksJdS)z.Check the keyword argument processing of pmap. cSg|]}t|ddqSrr;rYr}r,r,r.r/=z(test_parallel_kwargs..rrcs |] }tt|ddVqdSrrNr#rYr}r,r,r.rd>z'test_parallel_kwargs..Nr)r_lstr,r,r.test_parallel_kwargs9s  rcCstd}dd|D}td|d4}|j}||dd|Dks"J||dd|Dks/Jtdur>t|t|jus>JWdn1sHwYtdurZt|jdusZJ||d d|DksgJtdurtt|jdusvJdSdS) NrcSrrrr}r,r,r.r/Drz4test_parallel_as_context_manager..rr^csrrrr}r,r,r.rdMrz3test_parallel_as_context_manager..csrrrr}r,r,r.rdNrcsrrrr}r,r,r.rdZr)rhr"_backendrr8)r7rrpZmanaged_backendr,r,r. test_parallel_as_context_managerAs"rcsdGdddtttddtdddfdd td DWd d S1s+wYd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcStd)NZ123) RuntimeErrorrEr,r,r. __reduce__er=z.UnpicklableObject.__reduce__N)rMrNrOrr,r,r,r.UnpicklableObjectds rzthe task to sendmatchr@r0r^c3s|] }ttVqdSr9r#idrrr,r.rdis  z)test_parallel_pickling..rN)objectrrr"rhr,r,rr.test_parallel_pickling_s  "r byteorder)<>=csddtdd|djj}tdddfd d td D}|D]\}}||ks4J||jjks.inspect_byteorder)r@rfi4r@r0r^c3|] }tVqdSr9r#rrr:r,r.rdyrez5test_parallel_byteorder_corruption..rf) rZarangeZreshapeviewrrr"rhtestingassert_array_equal)rZinitial_np_byteorderresultZ x_returnedZbyteorder_in_workerr,rr."test_parallel_byteorder_corruptionms  rcCs0ttd|ddddtdDdksJdS)Nr@r_r7timeoutcsr)gMbP?Nrr}r,r,r.rdrez0test_parallel_timeout_success..r)rr"rhr6r,r,r.test_parallel_timeout_successs  rcCsNtttd|ddddtdDWddS1s wYdS)Nr@rrcsrrNrr}r,r,r.rdrez-test_parallel_timeout_fail..r)rr r"rhr6r,r,r.test_parallel_timeout_fails  "rc Cstdurtttd|dddtddDWdn1s$wYtttd|ddddDWdn1sEwYtd|d}t|jdusZJt|j}tt|ddtddDWdn1szwYt|jdusJt|j|usJd dtd D|d d td DksJt|j}tt|d ddDWdn1swYt|jdusJt|j|usJddtd D|dd td DksJ|j |j |j |j fWdn 1swYt|jdusJn!tttdddddDWdn 1s2wYtttddddtddDWdn 1sVwYtt tddddd tdDWddS1s{wYdS)Nr@r^cSg|] \}}tt||qSr,r#r<r-r:r;r,r,r.r/z&test_error_capture..)rr)rrcSg|]}tt|qSr,r#rWr}r,r,r.r/rcSrr,rrr,r,r.r/rcSrrrr}r,r,r.r/rrcsrrrr}r,r,r.rdrz%test_error_capture..cSrr,rr}r,r,r.r/rcSrrrr}r,r,r.r/rcsrrrr}r,r,r.rdrrrcSrr,rr}r,r,r.r/rcSrr,rrr,r,r.r/rr)r_rxcsr)T)rSNr#rTrbr,r,r.rdr)rrZeroDivisionErrorr"ziprVr8rrh _iteratingZn_completed_tasksZn_dispatched_tasks _abortingrC)r7r Zoriginal_workersr,r,r.test_error_capturest           $     $rc Csddd}td|dw}ttdd|dd |dd DWdn1s)wYttdd|d d |d d DWdn1sKwYttdd|d d |dd DWdn1smwYWddSWddS1swYdS)Nrcss*tdD] }||krtd|VqdS)NIterator Raising Error)rhrR)raise_atrcr,r,r. my_generators  z1test_error_in_task_iterator..my_generatorr@r^rrcsr`r9rarbr,r,r.rdz.test_error_in_task_iterator..)r csr`r9rarbr,r,r.rdr rcsr`r9rarbr,r,r.rdr r)r"rrR)r7r rr,r,r.test_error_in_task_iterators  "rcCs|d|dS)Nz Consumed %s)append)queueitemr,r,r.consumerrmrzbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) rrrrrrrrrrrrcsTtfdd}td||dfdd|D|ks Jtdks(JdS) zC Test that with only one job, Parallel does act as a iterator. c3(tdD] }d||VqdSNrz Produced %irhrrcrr,r.producer  z'test_dispatch_one_job..producerrr_ batch_sizer7c3s|] }tt|VqdSr9r#rr}r#r,r.rd rz(test_dispatch_one_job.. N)listr"r)r7r'Zexpected_queuer$r,r#r.test_dispatch_one_jobs  r+cst}|fdd}tddd|dfdd|Dt}|d d ks,J|d d d }|dks;J|d}||ksFJtdksNJd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3rr r!r"r#r,r.r$r%z/test_dispatch_multiprocessing..producerr@rrf)r_r' pre_dispatchr7c3s|] }ttdVqdS)rNr(rr#r,r.rd!rz0test_dispatch_multiprocessing..rrNrz Consumed anyrqrr))rManagerr*r"indexr)r7managerr$Zqueue_contentsZfirst_consumption_indexZproduced_3_indexr,r#r.test_dispatch_multiprocessings    r0cCs\tdddd}|ddtdD|jdksJWddS1s'wYdS) Nr@autor1r&csr`r9rrbr,r,r.rd8r z/test_batching_auto_threading..rr"rhrZcompute_batch_size)rr,r,r.test_batching_auto_threading2s"r4cCs\tdd|d}|ddtdD|jdksJWddS1s'wYdS)Nr@r1r&csr`r9rrbr,r,r.rd@r z2test_batching_auto_subprocesses..r2rr3)r7rr,r,r.test_batching_auto_subprocesses<s"r5cCsNtttddddddtdDWddS1s wYdS) zCMake sure that exception raised during dispatch are indeed capturedr@r)r_r,rxcsr`r9rrbr,r,r.rdKrez*test_exception_dispatch..rN)rrRr"rhr,r,r,r.test_exception_dispatchHs  "r7cC tddddtdDdS)Nr@rrcsr`r9rr-jr,r,r.rdPrez(nested_function_inner..rrgr"r,r,r.nested_function_innerO r;cCr8)Nr@rrcsr`r9)r#r;r9r,r,r.rdUrez(nested_function_outer..rrgr"r,r,r.nested_function_outerTr<r=z'https://github.com/joblib/loky/pull/255rcCstt}td|dddtdDWdn1swYt|j|j|j}d|}d|vs7Jd |vs=Jd |vsCJt|jtusLJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. r@r^csr`r9)r#r=rbr,r,r.rdcrez1test_nested_exception_dispatch..rNr=r;rT) rrRr"rhrr[rtbr)r7excinfoZ report_linesreportr,r,r.test_nested_exception_dispatchYs      rBc@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr9)r'r_r )rEr_r Z backend_argsr,r,r. configurets zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxrr(rEr_r,r,r.r'ysz$FakeParallelBackend.effective_n_jobsr)r)rMrNrOrPrDr'r,r,r,r.rCqs rCc Csttdd tddWdn1swYttddtddWdn1s2wYWdn1sAwYttdd&tddWdn1s]wYWddSWddS1suwYdS)NzInvalid backend:rz unit-testingr6)rrRr"r$r,r,r,r.test_invalid_backends    "rGcCs,tt}td|dWdn1swYdt|jvs%Jtt}td|dWdn1s.rcss |] \}}|t|kVqdSr9r|)r-rcrr,r,r.rdr)r"_effective_n_jobsrhr enumerate)r7r_rrr,r,r.test_njobs_converted_to_ints rNcCs<ztdtdtvs JtdtksJWtd=dStd=w)NZ test_backend)r&rCr)r,r,r,r.test_register_parallel_backends   rOcCsTttksJztdtdddttksJWtt_ntt_wttks(JdS)Nr1T)Z make_default)r]DefaultBackendr&r)rr*r r,r,r,r.test_overwrite_default_backends rQzOnly without multiprocessingcCsttddtddddtdDWdn1swYtddtddtdDWddS1sAwYdS) Nz)joblib backend '.*' is not available on.*rr0r6csr`r9rarbr,r,r.rdr z2test_backend_no_multiprocessing..rfcsr`r9rarbr,r,r.rdr )r UserWarningr"rhr$r,r,r,r.test_backend_no_multiprocessings "rScCs^||ddt\}}|dksJtddksJt}|jdks%J|dkr;t|tus1Jt|jtus:JnB|dkrQt|tusGJt|jtusPJn4|dkrgt|t us]Jt|jt usfJn.| drt|t ustJt|jt usJWddSWddSWddSWddSWddS1swYdS)Nrfrrr+r0r1Ztest_) r r\r'r"r_r[rrr!r startswithrC)rp backend_nameactive_backend active_n_jobsrr,r,r.check_backend_context_managers8    "rXcCsg|]}d|qS)ztest_backend_%dr,rbr,r,r.r/r~rfcsl|tvr |t|tttksJt|ttksJtdddfddtDttks4JdS)Nr@r1r^c3s"|] }|stt|VqdSr9)r#rX)r-rGrpr,r.rds   z/test_backend_context_manager..)r)setitemrCr]rPrXr" all_backends_for_context_manager)rr7rpr,rYr.test_backend_context_managers r\c@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)rRparam)rEr^r,r,r.rJs z%ParameterizedParallelBackend.__init__r9rLr,r,r,r.r]sr]cCs|tdtttksJ|dddd?t\}}t|tus$J|jdks+J|dks1Jt }|j dks;J|j |usBJ|ddt dD}Wdn1sWwY|dd t dDksiJttkspJdS) NZ param_backend*rf)r^r_csr`r9r#rrbr,r,r.rdr z=test_parameterized_backend_context_manager..rcSr{r,rrbr,r,r.r/r~z>test_parameterized_backend_context_manager..) rZr)r]r]rPr r\r[r^r"r_rrh)rrprVrWrrr,r,r.*test_parameterized_backend_context_managers    racCsttksJ|tdddd?t\}}t|tusJ|jdks&J|dks,Jt}|jdks6J|j |us=J|ddt dD}Wdn1sRwY|ddt dDksdJttkskJdS) N+)r^rrrcsr`r9r`rbr,r,r.rd(r zFtest_directly_parameterized_backend_context_manager..cSr{r,rrbr,r,r.r/)r~zGtest_directly_parameterized_backend_context_manager..) r]rPr]r r\r[r^r"r_rrh)rprVrWrrr,r,r.3test_directly_parameterized_backend_context_managers   rccCstdtS)Nr)rosgetpidr,r,r,r.sleep_and_return_pid/srfcCs<ttksJtdksJtddddtdDS)Nrr@rrcs|]}ttVqdSr9)r#rfrr,r,r.rd;sz"get_nested_pids..)r]rr"rLrhr,r,r,r.get_nested_pids4s rhcs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSrZ)superriget_nested_backendr __class__r,r.rkAszMyBackend.get_nested_backend)rMrNrOrPrk __classcell__r,r,rlr.ri?sriback_compat_backend)r1r0r+rocCsh||&tddddtdD}|D] }tt|dks!JqWddS1s-wYdS)Nr@rrcsrgr9)r#rhrr,r,r.rdRs  z6test_nested_backend_context_manager..rr)r"rhrset)rpr7Z pid_groupsZ pid_groupr,r,r.#test_nested_backend_context_managerIs "rq)r@rqNcs~ddtddfddtdD|dtddfddtdDWddS1s8wYdS) NcSs0tt|ks Jt|}t|ksJdSr9)r]r)r'r"rL)Zexpected_backend_typeZexpected_n_jobr,r,r.check_nested_backendbsz?test_nested_backend_in_sequential..check_nested_backendrrrc3s|] }ttdVqdSr)r#r*r)rrr,r.rdlrz4test_nested_backend_in_sequential..rc3s|] }tVqdSr9rrr7rrr_r,r.rdrrrg)r7r_rpr,rsr.!test_nested_backend_in_sequentialZs  "rtcCs\|| }|tur |d}|tur|d}|j|ksJWddS1s'wYdS)Nr7r)r$r% nesting_level)rp inner_backendZexpected_levelctxr7r,r,r.check_nesting_levelxs "rx outer_backendrvcst|dtd|dfddtdDddtfddtdDWddS1s9wYdS) Nrr@r^c3 |] }ttdVqdSrr#rxrrprvr,r.rds  z-test_backend_nesting_level..rrrc3rzrr{rr|r,r.rdr)rxr"rh)rpryrvr,r|r.test_backend_nesting_levels  "r}with_retrieve_callbackTcsddlGfdddt}td|dd|d;}tddfd d td D|tur:|d jd ks:J|turG|djd ksOJWddSWddS1sZwYdS)Nrcs"eZdZdZZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrsr"rr,r,r.retrieval_contexts z;test_retrieval_context..MyBackend.retrieval_contextN)rMrNrOrcZsupports_retrieve_callbackcontextmanagerrr,) contextlibr~r,r.ris riZ retrievalcSstddddt|DS)Nr@rrcsr`r9rrbr,r,r.rdr z>test_retrieval_context..nested_call..rg)nr,r,r. nested_callsz+test_retrieval_context..nested_callr@rrc3s|] }t|VqdSr9rrb)rr,r.rds  z)test_retrieval_context..rr7r)rrr&r"rhr$rcr%)rpr~rirwr,)rrr~r.test_retrieval_contexts   "rr')rrqgQ?cCs8ttt|dWddS1swYdS)Nr')rrRr"rr,r,r.test_invalid_batch_sizes  "rz)n_tasks, n_jobs, pre_dispatch, batch_size) )r@r@rr1)r@r@r_r1rr@r_r1)ir@r_r1r)rrr_r1)r)r_r1)r) 2 * n_jobsr)r)rr)rr)rrQ)rr)rr1cCsN|||d}ddt|D}tdi|ddt|D}||ks%JdS)N)r_r,r'cSr{r,r|rbr,r,r.r/r~z0test_dispatch_race_condition..csr`r9rarbr,r,r.rdr z/test_dispatch_race_condition..r,r)Zn_tasksr_r,r'paramsrrr,r,r.test_dispatch_race_conditions  rcCs8t}tddd}|jd}|}||ksJdS)Nr@r+r^rp)rget_start_methodr"Z _backend_argsget)Zmp_start_methodrrpZ start_methodr,r,r.test_default_mp_contexts   rcs^|dkr td}tjd}|ddtjtd|dfddt dDdS) Nr+r3r_ir@r^c3s"|] }ttjjVqdSr9)r#rdotTrbrFr,r.rds zAtest_no_blas_crash_or_freeze_with_subprocesses..) rr2rrandomZ RandomStaterandnrrr"rh)r7rngr,rr..test_no_blas_crash_or_freeze_with_subprocessess    rafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr+forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) rrrskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrr executable)r7coder,r,r.2test_parallel_with_interactively_defined_functionss    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r#rkwargsc Csv|dvr|tks tjdkrtdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r+r3win32zNot picklable with pickle)rr7rjoblib_root_folderzunpicklable_func_script.pyrrr) SQUARE_MAINrplatformrr)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrrdpathdirnamejoblib__file__rwriterrstrpath)r7rrtmpdirrZ code_filer,r,r.0test_parallel_with_unpicklable_functions_in_argsCs"     raimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyr)rr)rr5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrrrscriptr,r,r.7test_parallel_with_interactively_defined_functions_lokys    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]Z LokyProcess)rZ stderr_regexr)rr9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrrrr,r,r.:test_parallel_with_interactively_defined_bound_method_lokys    rcCs"tg}tdd|gksJdS)Nr@rr)iterr")Zexhausted_iteratorr,r,r.%test_parallel_with_exhausted_iteratorsrcCsttddS)z+Helper function to force gc in each worker.rN)rrUrr,r,r,r._cleanup_workersrcCs"t|tjs tdt||S)Nz#Expected np.memmap instance, got %r) isinstancermemmap TypeErrorr[copyrr,r,r. check_memmaps rcCsdd}tdd|ddd|dD}t||t|D] \}}tj||qtd d|dd d|dD}t||t|D] \}}tj||qBdS) Ncss*t|D] }tjdtjd|VqdS)Nrr)rhronesZfloat32)rrcr,r,r.generate_arrayss zBtest_auto_memmap_on_arrays_from_generator..generate_arraysr@rr_ max_nbytesr7csr`r9r#rr-rFr,r,r.rdrez.rzrcsr`r9rrr,r,r.rdre)r"rrrrr)r7rrrrr,r,r.)test_auto_memmap_on_arrays_from_generatorsrcCs|Sr9r,)argr,r,r.identityrKrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj s;Jd j |ksDJtj ||dS) Nz test.mmapuint8rrK)Z mmap_moder@rrc3rr9)r#rrrr,r.rdr z.test_memmap_with_big_offset..r r)rrmmapZALLOCATIONGRANULARITYrzerosrr r r"rroffsetrr)rfnamesizeobjrr,rr.test_memmap_with_big_offsets   rcCstjdd}tdddddtdDWdn1s wYt|dks-J|d}t|jts9Jt|jd ksBJdS) NTrr)r_rcsr`r9rarbr,r,r.rdr zFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr"rhrrrrRr)rrr,r,r.3test_warning_about_timeout_not_supported_by_backends  rcCs |||<|Sr9r,) input_listr.rr,r,r.set_list_valuesr)rr@rcsVdgdt|dddfddtdD}t|}tddt|Ds)JdS) Nrr generatorr1r_ return_asr7c3s |] }tt||VqdSr9)r#rrbrr,r.rd  zPtest_parallel_return_order_with_return_as_generator_parameter..cs|] \}}||kVqdSr9r,r-vrKr,r,r.rdr )r"rhr*rr)r_rr,rr.=test_parallel_return_order_with_return_as_generator_parameters   rcCs|rtdt|S)Nr)rr)edelayr,r,r._sqrt_with_delaysrcsvt|d|dddtdDtfddtdD}dgttd d}td dt||Ds5JtdS) Ngenerator_unorderedrcss&|]}tt|d|dkVqdS)r@rN)r#rrbr,r,r.rd s zK_test_parallel_unordered_generator_returns_fastest_first..rc3s|]}tVqdSr9)nextrrr,r.rd#s rr@csrr9r,rr,r,r.rd're)r"rhsortedr*rrr)r7r_Zquickly_returnedZexpected_quickly_returnedr,rr.8_test_parallel_unordered_generator_returns_fastest_firsts  rr@ sequentialcCst||dSr9)rr7r_r,r,r.7test_parallel_unordered_generator_returns_fastest_first/srzThis test requires daskc Csvtjddd*|d td|Wdn1swYWddSWddS1s4wYdSNr@Z n_workersZthreads_per_workerdask) distributedClientr)r_rpr,r,r.Atest_parallel_unordered_generator_returns_fastest_first_with_dask9s Pr)rr@rwrqcCsvdgdgd}ttt}t||ddd|DWdn1s(wYt|}|dks9JdS)NrFrrzr^css|] }ttj|VqdSr9r#rUrrbr,r,r.rdKrz%test_abort_backend..r)rrrUr")r_r7Zdelayst_startdtr,r,r.test_abort_backendEs   rcCstjtdtd}d|d<|S)NgArFr)rrintbool)rrr,r,r.get_large_objectQsrcCsbt|||d }|ddtdD}t|t|~tWddS1s*wYdS)N)r_r7rcsr`r9)r#rrbr,r,r.rd\r z0_test_deadlock_with_generator..r)r"rhrr)r7rr_r rr,r,r._test_deadlock_with_generatorWs"rrrrcCst|||dSr9)r)r7rr_r,r,r.test_deadlock_with_generatoresrc Csxtjddd+|dtd||Wdn1swYWddSWddS1s5wYdSr)rrr)rprr_r,r,r.%test_deadlock_with_generator_and_daskmsPrcCsttdd)t|||d}|ddtdD}t}|ddtdD}Wdn1s2wYt|d ksCJd ~tdS) N)This Parallel instance is already runningrr7rcsrrrrr,r,r.rdr z/test_multiple_generator_call..rcsr`r9rrbr,r,r.rdr rzr@ZThe error should be raised immediatly when submitting a new task but it took more than 2s.)rrr"rhrUr)r7rr_r grZgen2r,r,r.test_multiple_generator_callys rc Cst|||dD}|ddtdD}t}ttdd|ddtdD}Wdn1s3wYt|d ksDJd Wdn1sNwY~tdS) Nrcsrrrrr,r,r.rdr z7test_multiple_generator_call_managed..rrrcsr`r9rrbr,r,r.rdr rzr@r)r"rhrUrrr)r7rr_r rrg2r,r,r.$test_multiple_generator_call_manageds" r return_as_1 return_as_2cCst|||dddtdD}t|||dddtddD}|dkr)t|}|dkr1t|}tddt|tdDsAJtd dt|tddDsRJdS) Nrcs |] }tt|dVqdSr@Nr`rbr,r,r.rdrz9test_multiple_generator_call_separated..rcsrrr`rbr,r,r.rdrrrcsrr9r,r-rrcr,r,r.rdr csrr9r,rr,r,r.rdr )r"rhrrr)r7rrr_rrr,r,r.&test_multiple_generator_call_separateds &r zbackend, error))r0T)r1F)rFc Csb|dkr tdur tdtd||d}|ddtdD}t|dd }|r/ttd d nt }|9t }td||dd dtdd D}t |dkrUt |}t ddt|tdd DsfJWdn1spwYt |dksJd} |jr| dkr| d7} t d|jr| dkst |dksJ|dkr|jsJdSdS)Nr0zRequires multiprocessingr@rcsrrrrbr,r,r.rdr z.rcSstdS)NzGenerator collected)rr,r,r,r.z;test_multiple_generator_call_separated_gc..z The executor underlying Parallelrcsrrr`rbr,r,r.rdrrrcsrr9r,rr,r,r.rdr rrrfrrH)rrrr"rhweakreffinalizerrrrUrrrraliverrLr) r7rrerrorr rZg_wrrwrretryr,r,r.)test_multiple_generator_call_separated_gcs< $   rcCs |j}tdd||d.}|ddtjdgdDtt|dks'J|ddtdDWdn1s.rrcsrgr9r#rrr,r,r.rdrzrz/temporary directory of Parallel was not removedrcsr`r9rrr,r,r.rd#r csrgr9rrr,r,r.rd$r) rr"rrrrdlistdirrhrAssertionError)r7rrrr,r,r.test_memmapping_leakss,        r)Nr0r1cCs<td|dddtdD}|ddtdDksJdS)Nr@r^css |] }tdd|VqdS)cSr>r?r,rAr,r,r.r 5r z2test_lambda_expression...Nrrbr,r,r.rd4rz)test_lambda_expression..rcSsg|]}|dqS)r@r,rbr,r,r.r/6r~z*test_lambda_expression..rg)r7rr,r,r.test_lambda_expression.srcsd}d}d|td||d}|fddt|D|jj|jjks&J|jj|jjks0J|fddt|D|jj|jjksGJ|jj|jjksQJd S) zCTest that a parallel backend correctly resets its batch statistics.r@ig@r)rxr_r7c3|] }ttjVqdSr9rrbZ task_timer,r.rdBz6test_backend_batch_statistics_reset..c3rr9rrbrr,r.rdHrN)r"rhrZ_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r7r_Zn_inputsrr,rr.#test_backend_batch_statistics_reset9s$ rcCsdD];}tt|djtksJt|dd}t|jtusJt|dd}t|jtus.Jt|dd}t|jtus=Jqtdd dd }t|jtusNJ|d ddtdd }t|jtuscJ|jdksjJWdn1stwY|d ddtd dd}t|jtusJ|jd ksJWdn1swY|d ddtdd }t|jtusJ|jdksJWdn1swY|d ddtd dd}t|jtusJ|jd ksJWddS1swYdS)N)rr@rqrrthreadsr_prefer processes sharedmemr_requirer@r0)r_r7rrrfr"r)r[r"rrPrr!r_)rpr_rr,r,r.$test_backend_hinting_and_constraintsOs<       "r%cCszGdddt}||&tddd}t|j|usJtddd}t|j|us,JWdn1s6wYGdd d t}||Otddd}t|j|usXJ|\}}|d ksdJ|d ksjJtddd d }t|jtuszJ|\}}d }||ksJ|d ksJWdn1swYttt|ddWddS1swYdS)Nc@$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSrDr9r,rr,r,r. apply_asyncrKzgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccS|Sr9r,rFr,r,r.r'rKzltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrMrNrOZsupports_sharedmemZ use_threadsr'r'r,r,r,r.MyCustomThreadingBackend  r*r@rrr r!c@r&)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSrDr9r,rr,r,r.r'rKzhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSr(r9r,rFr,r,r.r'rKzmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNr)r,r,r,r.MyCustomProcessingBackendr+r,r>r)r_r"rxzUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r7r") r r"r[rZ readouterrrstriprrR)Zcapsysrpr*rr,outerrrr,r,r.9test_backend_hinting_and_constraints_with_custom_backendss0        "r1cCstt tddWdn1swYtt tddWdn1s-wYtttdddWdn1sGwYtdurtttdddWdn1sewYtttd ddWddS1swYdSdS) Ninvalidr#r$rr )rr"r0r-r+)rrRr"rr,r,r,r.,test_invalid_backend_hinting_and_constraintss"       "r3c stdd4}t|jj|jjfg}dkr|WdS|fddtdD}||dWdS1s.r)r"r[rrMrurh)r4rrZ this_levelrr,r6r.r5s  $r5cCs||dd t}Wdn1swYtdkr#d}d}nd}d}|d}|df|dfd|fd|fg}||ksAJdS)Nr@rrrrrZBackendr)r5r(title)rpr7backend_types_and_levelsZsecond_level_backend_typeZ max_levelZtop_level_backend_typeZexpected_types_and_levelsr,r,r.test_nested_parallelism_limits  r9c Cstjdddptjtdtjd}tdD].}|d t|d}Wdn1s,wYt|dks9Jt dd |DsDJq|d t}Wdn1sWwYt|dksdJt d d |DsoJWddS1szwYdS) Nr@rgcArr)datarcs|] \}}|dkVqdSZDaskDistributedBackendNr,r-rrr,r,r.rdz4test_nested_parallelism_with_dask..csr;r<r,r=r,r,r.rdr>) rrrrrrrhr5rr)rpr:rcr8r,r,r.!test_nested_parallelism_with_dasks$    "r?cCstddtdDS)z6A horrible function that does recursive parallel callscsrgr9)r#_recursive_parallelrbr,r,r.rdrz&_recursive_parallel..r@Nrg)Z nesting_limitr,r,r.r@sr@c Cs||ddtt }tWdn1swYWdn1s(wY|j}|dkrFddlm}t||tfrFt dt|t sMJdS)Nr@rrr0r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionr@rZ&joblib.externals.loky.process_executorrArrrrRecursionError)rpr7r@excrAr,r,r.test_thread_bomb_mitigation s   rEcCs*i}dD] }tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADS ENABLE_IPCrz)rdenvironrr)Zenv_varsvarr,r,r._run_parallel_sum's rLzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]-\}}||ks#J|D]\}}|dr9|t|ks8Jq'|dks?J|dksEJq'qdS) Nr@r^csrgr9)r#rLrr,r,r.rd3  z-test_parallel_thread_limit..rZ_THREADSrI1)r"rhrEr(itemsendswithr)r7rZexpected_num_threadsZworker_env_varsZomp_num_threadsrrr,r,r.test_parallel_thread_limit0s    rQcCs:ttdd |dWddS1swYdS)NzPlease install daskrr)rrRrYr,r,r.)test_dask_backend_when_dask_not_installedAs "rRc CsGdddt}d}||6tjt|dtddddtdDWdn1s0wYWddSWddS1sHwYdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_dSrZr,)rErrr,r,r.rDMrKz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSr)NzNo worker availabler)rEfunccallbackr,r,r.r'Pr=z?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSrSrZr,rFr,r,r.r'SrKzDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsr9)rMrNrOrDr'r'r,r,r,r.ZeroWorkerBackendLs  rVz&ZeroWorkerBackend has no active workerrr@rrcsr`r9rrbr,r,r.rdYr z+test_zero_worker_backend..)rrrrr"rh)rprVZ expected_msgr,r,r.test_zero_worker_backendHs "rWcsdadddks JtddfddtdD}t|dhks&Jdadks/Jtddfd dtdD}t|dhksHJdS) Nzoriginal valuecSstSr9)MY_GLOBAL_VARIABLEr,r,r,r. check_globalsdsz@test_globals_update_at_each_parallel_call..check_globalsr@rrc3|]}tVqdSr9rrbrYr,r.rdjrMz.z changed valuec3rZr9rrbr[r,r.rdsrM)rXr"rhrp)Zworkers_global_variabler,r[r.)test_globals_update_at_each_parallel_call\sr\cCs4ddl}|jdd}|||ddlm}|S)Nrrz)threadpool_info)ZnumpyrrrZ threadpoolctlr])rrFr]r,r,r._check_numpy_threadpool_limits}s   r^cCs4|D]}|d|dkr|dSqtd|)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rRr) child_module parent_info parent_moduler,r,r._parent_max_num_threads_fors rdcCs@|D]}|D]}t||}t|||h}|d|vsJqqdS)Nr`)rdmin)Z workers_inforbr`Zchild_threadpool_inforaZparent_max_num_threadsrr,r,r.check_child_num_threadssrf)r@rrwrqcCsdt}t|dkrtjddtd|dddtdD}t|}tt|d }t |||dS) Nr&Need a version of numpy linked to BLASrr0rcsrgr9r#r^rbr,r,r.rdrMz;test_threadpool_limitation_in_child_loky..r@r r^rrrr"rhr'rEr(rf)r_rbworkers_threadpool_infosexpected_child_num_threadsr,r,r.(test_threadpool_limitation_in_child_lokys  rlinner_max_num_threads)rr@rNcCst}t|dkrtjdd|d|dt|dddtd D}Wdn1s.wYt|}|durDtt|d }n|}t |||dS) Nrrgrr0rmrrcsrgr9rhrbr,r,r.rdrMz>test_threadpool_limitation_in_child_context..r@rri)rpr_rmrbrjrkr,r,r.+test_threadpool_limitation_in_child_contexts   rovar_name)rGrHrFc stddddtj}z_dtj<t|dfddtd D}|ddgks0J|d d d t|dfd dtd D}Wdn1sRwY|ddgks_JW|durjtj=dS|tj<dS|durztj=w|tj<w)NTZreusecSs tj|Sr9)rdrJr)rpr,r,r._get_envrz>test_threadpool_limitation_in_child_override.._get_env4rrc3rr9rrbrrrpr,r.rdrez?test_threadpool_limitation_in_child_override..r@r0rrnc3rr9rrbrtr,r.rdrerN)rshutdownrdrJrr"rh)rpr_rpZoriginal_var_valuerr,rtr.,test_threadpool_limitation_in_child_overrides*     rv)r@rrqcCsJdd}||tdd}tdD]}||tdd}||ks"JqdS)Ncs,tdt|dfddtdDdS)Nrrrc3rr9)r#sumrbrAr,r.rdr zAtest_loky_reuse_workers..parallel_call..rrrr,rAr. parallel_calls$z.test_loky_reuse_workers..parallel_callTrqr)rrh)r_rxZfirst_executorrexecutorr,r,r.test_loky_reuse_workerss   rz)F)rr)rfr9)rPrdrrUrr rr1 tracebackrmathrrpicklerrrr+r rrr r r Zjoblib._multiprocessing_helpersrZjoblib.test.commonrrrrrZjoblib.testingrrrrrZjoblib.externals.lokyrrrposix ImportErrorZ _openmp_test_helper.parallel_sumrrZjoblib._parallel_backendsrrrr r!Zjoblib.parallelr"r#r$r%r&r'r(r)r*rZRETURN_GENERATOR_BACKENDSpoprkeysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDShasattrrr2rPr8r<rB ExceptionrCrTrWrYr]rjrlromarkrvrrrrrrrrrrrrrrrrrrrr+r0r4r5r7r;r=rrBrCrGrIrNrOrQrSrXr[extendrhr\r]rarcrfrhZ_parallel_backendsrirqrtrxr}rrrrrrrrrZ SQUARE_LOCALZ SQUARE_LAMBDArrrrrrrrrrrrrrrrrrrrrprrrrrrrrrr rrrrr%r1r3r5r9r?r@Zno_coverrErLrQrRrWr\r^rdrfrlrorvrzr,r,r,r.s                                                 D                                   ( ,             2+  . 6          !