o i/@sd dZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddlm Z ddl m Z ddlmZddl Z ddlZddlZddlZddlmZdd lmZmZdd lmZdd lmZmZdd lmZdd lmZmZmZmZm Z edurddl!m"Z"ddl#m$Z$zddl%Z%Wn e&ydZ%Ynwzddl'm(Z(Wn e&ydZ(Ynwzddl)Z)Wn e&ydZ)Ynwddl*m+Z+ddl*m,Z,ddl*m-Z-ddl*m.Z.ddl*m/Z/ddl*m0Z0ddl1m2Z2m3Z3ddl1m4Z4m5Z5ddl1m6Z6m7Z7ddl1mZm8Z8m9Z9m:Z:ddl;me8?Z@e@dde8D7Z@edur.gZAnd d!gZAeAd"gZBeCed#rEe@DeEd$e8e9ZFd%d&ZGd'd(ZHd)d*ZIGd+d,d,eJZKd'd.d/ZLd0d1ZMd(d2d3ZNd4d5ZOd6d7ZPd8d9ZQd:d;ZRejSjde6d>d?fd@ggdAdBedCdDZTedEe@edFgdGedHgdIdJdKZUedEe@dLdMZVdNdOZWeedPgdQdRdSZXeedEgdTdUdVZYdWdXZZedYe8edZe8d[d\Z[d]d^Z\ed_d`Z]dadbZ^edFgdcdddeZ_edEeBdfdgZ`edhdiZaedEeBdjdkZbeedEeBdldmZceedEeAdndoZddpdqZeedEe8edrdgdsfdtgdufgdvdwZfeedEeBdxdyZgdzd{ZheedEeAd|d}Zid~dZjddZkddZleedEeBejSjmddddZnGddde+ZoddZpedEe@ddZqddZrddZseeduddddZtddZueBddZvevwddexdDeedEevddZyGddde+ZzddZ{ddZ|ddZ}ddZ~Gdddejj/Ze4deeedEgdddZeedFgdedEeBddZddZeedeBedeBddZeddZddZddZedgdddZedgdddZeddÄZeeedEeAddńZdZeedEeAddȄZdZdZdZdZeedEeAedur|gnd$gedeeegedgdϢddфZdjejejejdӍZeddՄZdjejejejdӍZedd؄ZddڄZdd܄ZeeedEeAddބZddZeeddZddZedEe@edFgdddZeeedEeAddZedEedur dd"gngdddZeedEeAddZeddZddZddZddZddZd)ddZeedEd!d"gddZeee)duddddZd*ddZedEedurid"gnd!d"gddZddZedEedurdd!gndgee(duddddZee)dud dd d Zd d ZddZddZddZddZeeedFgdddZeeedgdedFdd>gddZeedFdd>gedgdd d!ZeeedEd d"e-e,gd"d#ZeedFgd$d%d&ZdS(+z Test the parallel module. N)format_exceptionsqrt)sleep) PicklingError TimeoutErrorparallel)dumpload)mp)np with_numpy)with_multiprocessing) parametrizeraisescheck_subprocess_callskipifwarns)get_reusable_executor)Queue) parallel_sum)SequentialBackend)ThreadingBackend)MultiprocessingBackend)ParallelBackendBase) LokyBackend) SafeFunction)Paralleldelayed)register_parallel_backendparallel_backend)effective_n_jobs cpu_count)r BACKENDSDEFAULT_BACKENDEXTERNAL_BACKENDS)JoblibException)WorkerInterruptcCsg|]}t|qS)r&).0 backend_strr+r+T/var/www/edux/Edux_v2/venv/lib/python3.10/site-packages/joblib/test/test_parallel.py Jsr/multiprocessingloky threading get_contextspawncCst|dt|ddS)N_pool_workers)getattrbackendr+r+r. get_workersXsr:cCs||SNr+)xyr+r+r.division\r>cC|dSNr+r<r+r+r.square`r?rDc@seZdZdZddZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cCdSr;r+)selfabcdr+r+r.__init__gz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rLr+r+r+r.rEds rEFcCs"|dkr|r tddddt|S)NrHrIrJrK)rE ValueError)r<custom_exceptionr+r+r.exception_raiserks rVcCstdt)Ng?)timerKeyboardInterruptrCr+r+r.interrupt_raiserrs rYcCs|d||S)zO A module-level function so that it can be spawn with multiprocessing. rBr+)r<r=zr+r+r.fwsr[cCsttdSNr)typer get_active_backendr+r+r+r._active_backend_type~sr_cCst||dddtdDS)Nn_jobsr9cs|] }tt|VqdSr;r!rDr,ir+r+r.   z parallel_func..r range) inner_n_jobsr9r+r+r. parallel_funcsrlcCtdksJdSr\)r%r+r+r+r.test_cpu_countrncCrmr\)r$r+r+r+r.test_effective_n_jobsrorpzbackend_n_jobs, expected_n_jobs)rhrhraNr)z positive-intz negative-intNone)idscCsTtd|dtdd|ksJWdn1swYtdddks(JdS)Nr2rrr)r#r$)backend_n_jobsexpected_n_jobsr+r+r.test_effective_n_jobs_Nones rxr9ra)rrBrqverbose)rB dcCs:ddtdDt|||dddtdDksJdS)NcSg|]}t|qSr+rDr,r<r+r+r.r/z(test_simple_parallel..)rar9rzcsrbr;rcrr+r+r.rfrgz'test_simple_parallel..rjr )r9rarzr+r+r.test_simple_parallels rcCs|jtdddtjdd}td|ddd td D}|gd ks(JWdn1s2wYd d |D}t|dksFJdS)Nname!some_new_name_for_the_main_thread)targetrvalueTrecordrBr`csrbr;rcrr+r+r.rfrgz6test_main_thread_renamed_no_warning..rh)rrcSsg|] }dt|jvr|qS)zworker timeout)strmessager,wr+r+r.r/z7test_main_thread_renamed_no_warning..r)setattrr2current_threadwarningscatch_warningsr rjlen)r9 monkeypatchwarninforesultsr+r+r.#test_main_thread_renamed_no_warnings rcCstjdd}tdt||dWdn1swYdd|D}|r=|r;t|dko:d|d jd vSd S|rAJdS) NTralways)r9rkcSsg|]}|jqSr+)rrr+r+r.r/sz*_assert_warning_nested..rzbacked parallel loops cannotrF)rr simplefilterrlrargs)r9rkexpectedrr+r+r._assert_warning_nesteds  rz%parent_backend,child_backend,expected))r1r0T)r1r1F)r0r0T)r0r1T)r2r0T)r2r1Tcsntd|dfddtdDtd|dfddtdD}|dkr/t|s-JdSt|s5JdS)NrBr`c3s"|] }ttdddVqdS)rFr9rkrNr!rr,_ child_backendr+r.rf z0test_nested_parallel_warnings..rc3s"|] }ttddVqdS)rBrNrrrrr+r.rfrr2)r rjanyall)parent_backendrrresr+rr.test_nested_parallel_warningss r)r1r0r2cCs>dg}dd}tj||fd}|||dsJdS)NFcSshtjdd}tddddtdDWdn1swYtt|t|dk|d<dS) NTrrBrrcs|] }ttdVqdS)皙?Nr!rrr+r+r.rfrgzPtest_background_thread_parallelism..background_thread..rr)rrr rjprintr)is_run_parallelrr+r+r.background_threads  z=test_background_thread_parallelism..background_thread)rrr)r2Threadstartjoin)r9rrtr+r+r."test_background_thread_parallelisms rcCs"td|dddtdDdS)NrBr`csr){Gz?Nrcrr+r+r.rf rgznested_loop..rir8r+r+r. nested_loop s rrrcs&td|dfddtdDdS)NrBr`c3|] }ttVqdSr;)r!rrrr+r.rfrgz#test_nested_loop..ri)rrr+rr.test_nested_loops rcCstr;)rTr8r+r+r.raise_exceptionrMrc Cstt5tddd}|ttdttdgWdn1s$wYWddSWddS1s.N)rr rjfullr+r+rr.test_mutate_input_with_threads$s r)rrBrhcCs6td}dd|Dt|ddd|DksJdS)z.Check the keyword argument processing of pmap. cSg|]}t|ddqSrr=r[rr+r+r.r/0z(test_parallel_kwargs..rrcs |] }tt|ddVqdSrrNr!r[rr+r+r.rf1z'test_parallel_kwargs..Nr)ralstr+r+r.test_parallel_kwargs,s  rcCstd}dd|D}td|d4}|j}||dd|Dks"J||dd|Dks/Jtdur>t|t|jus>JWdn1sHwYtdurZt|jdusZJ||d d|DksgJtdurtt|jdusvJdSdS) NrcSrrrrr+r+r.r/7rz4test_parallel_as_context_manager..rr`csrrrrr+r+r.rf@rz3test_parallel_as_context_manager..csrrrrr+r+r.rfArcsrrrrr+r+r.rfMr)rjr _backendr r:)r9rrpmanaged_backendr+r+r. test_parallel_as_context_manager4s"rcsbGdddtttddtddfddtd DWd d S1s*wYd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcStd)N123) RuntimeErrorrGr+r+r. __reduce__Xr?z.UnpicklableObject.__reduce__N)rOrPrQrr+r+r+r.UnpicklableObjectWs rzthe task to sendmatchrBrrc3s|] }ttVqdSr;r!idrrr+r.rf\z)test_parallel_pickling..rN)objectrrr rjr+r+rr.test_parallel_picklingRs""rcCs0ttd|ddddtdDdksJdS)NrBrrar9timeoutcsr)gMbP?Nrrr+r+r.rfbrgz0test_parallel_timeout_success..)rr rjr8r+r+r.test_parallel_timeout_success_s  rcCsNtttd|ddddtdDWddS1s wYdS)NrBrrcsr)rNrrr+r+r.rfkrgz-test_parallel_timeout_fail..r)rrr rjr8r+r+r.test_parallel_timeout_failfs  "rc CsJtdur tttd|dddtddDWdn1s$wYtttd|ddddDWdn1sEwYtd|d}t|jdusZJt|j}tt|ddtddDWdn1szwYt|jdusJt|j|usJd dtd D|d d td DksJt|j}tt|d ddDWdn1swYt|jdusJt|j|usJddtd D|dd td DksJWdn1swYt|jdus Jn!tt tdddddDWdn 1s'wYtttddddtddDWdn 1sKwYtt tddddd tdDWdn 1sowYztdddd tddDWtdt y}zt |t rJWYd}~dSd}~ww)NrBr`cSg|] \}}tt||qSr+r!r>r,r<r=r+r+r.r/wz&test_error_capture..)rr)rrcSg|]}tt|qSr+r!rYrr+r+r.r/{rcSrr+rrr+r+r.r/rcSrrrrr+r+r.r/rrcsrrrrr+r+r.rfrz%test_error_capture..cSrr+rrr+r+r.r/rcSrrrrr+r+r.r/rcsrrrrr+r+r.rfrrrcSrr+rrr+r+r.r/rcSrr+rrr+r+r.r/rr)rarzcsr)T)rUNr!rVrdr+r+r.rfsrcss"|] \}}tt||VqdSr;rrr+r+r.rf z'The excepted error has not been raised.)r rZeroDivisionErrorr zipr*r:rrjrXrE Exception isinstancer)rT)r9r original_workersexr+r+r.test_error_captureos~            !      rcCs|d|dS)Nz Consumed %s)append)queueitemr+r+r.consumerrorzbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) rrrrrrrrrrrrcsTtfdd}td||dfdd|D|ks Jtdks(JdS) zC Test that with only one job, Parallel does act as a iterator. c3(tdD] }d||VqdSNz Produced %irjrrerr+r.producer  z'test_dispatch_one_job..producerrra batch_sizer9c3s|] }tt|VqdSr;r!rrr r+r.rfrz(test_dispatch_one_job.. N)listr r)r9rexpected_queuer r+r r.test_dispatch_one_jobs  rcst}|fdd}tddd|dfdd|Dt}|d d ks,J|d d d }|dks;J|d}||ksFJtdksNJd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3rrr r r r+r.r r z/test_dispatch_multiprocessing..producerrBrrh)rar pre_dispatchr9c3|] }ttdVqdS)rNrrr r+r.rfrz0test_dispatch_multiprocessing..rrNrz Consumed anyrqrr)r Managerrr indexr)r9managerr queue_contentsfirst_consumption_indexproduced_3_indexr+r r.test_dispatch_multiprocessings    rcCs\tdddd}|ddtdD|jdksJWddS1s'wYdS) NrBautor2rcsrbr;rrdr+r+r.rfz/test_batching_auto_threading..rr rjrcompute_batch_size)rr+r+r.test_batching_auto_threadings"r#cCs\tdd|d}|ddtdD|jdksJWddS1s'wYdS)NrBrrcsrbr;rrdr+r+r.rfrz2test_batching_auto_subprocesses..r rr!)r9rr+r+r.test_batching_auto_subprocessess"r$cCsNtttddddddtdDWddS1s wYdS) zCMake sure that exception raised during dispatch are indeed capturedrBr)rarrzcsrbr;rrdr+r+r.rfrgz*test_exception_dispatch..rN)rrTr rjr+r+r+r.test_exception_dispatchs  "r&cC tddddtdDdS)NrBrrcsrbr;rr,jr+r+r.rfrgz(nested_function_inner..rrir r+r+r.nested_function_inner r*cCr')NrBrrcsrbr;)r!r*r(r+r+r.rfrgz(nested_function_outer..rrir r+r+r.nested_function_outerr+r,z'https://github.com/joblib/loky/pull/255)reasoncCstt}td|dddtdDWdn1swYt|j|j|j}d|}d|vs7Jd |vs=Jd |vsCJt|jtusLJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. rBr`csrbr;)r!r,rdr+r+r.rf+rgz1test_nested_exception_dispatch..rNr,r*rV) rrTr rjrr]rtbr)r9excinfo report_linesreportr+r+r.test_nested_exception_dispatch!s      r3c@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr;)r$rar )rGrar backend_argsr+r+r. configure<s zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxr r%rGrar+r+r.r$Asz$FakeParallelBackend.effective_n_jobsr)r)rOrPrQrRr6r$r+r+r+r.r49s r4c Cstt }tddWdn1swYdt|jvs"Jtt}tdWdn1s6wYWdn1sEwYdt|jvsSJdS)Nz unit-testingr8zInvalid backend:)rrTr rrr#)r0r+r+r.test_invalid_backendGs    r9cCsNtt}td|dWdn1swYdt|jvs%JdS)Nrr`z&n_jobs == 0 in Parallel has no meaning)rrTr _initialize_backendrr)r9r0r+r+r.test_invalid_njobsRs r;cCs<ztdtdtvs JtdtksJWtd=dStd=w)N test_backend)r"r4r&r+r+r+r.test_register_parallel_backendYs   r=cCsTttksJztdtdddttksJWtt_ntt_wttks(JdS)Nr2T) make_default)r_DefaultBackendr"r&rr'r r+r+r+r.test_overwrite_default_backendbs r@zOnly without multiprocessingcCsttddtddddtdDWdn1swYtdtddtdDWddS1s@wYdS) Nz)joblib backend '.*' is not available on.*rr1r8csrbr;rcrdr+r+r.rfrrz2test_backend_no_multiprocessing..rhcsrbr;rcrdr+r+r.rfvr)r UserWarningr rjr#r+r+r+r.test_backend_no_multiprocessingns "rBcCs^t|ddt\}}|dksJtddksJt}|jdks%J|dkr;t|tks1Jt|jtks:JnB|dkrQt|t ksGJt|jt ksPJn4|dkrgt|t ks]Jt|jt ksfJn.| drt|t kstJt|jt ksJWddSWddSWddSWddSWddS1swYdS)Nrhrrr0r1r2test_) r#r r^r$r rar]rrrr startswithr4) backend_nameactive_backend active_n_jobsrr+r+r.check_backend_context_managerys8    "rHcCsg|]}d|qS)ztest_backend_%dr+rdr+r+r.r/rrhcCsf|tvr |t|tttksJt|ttksJtdddddtDttks1JdS)NrBr2r`css |] }|stt|VqdSr;)r!rH)r,rIr+r+r.rfs   z/test_backend_context_manager..)r&setitemr4r_r?rHr all_backends_for_context_manager)rr9r+r+r.test_backend_context_managersrKc@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)rTparam)rGrMr+r+r.rLs z%ParameterizedParallelBackend.__init__r;rNr+r+r+r.rLsrLcCs|tdtttksJtdddd?t\}}t|tks$J|j dks+J|dks1Jt }|j dks;J|j |usBJ|ddt dD}Wdn1sWwY|dd t dDksiJttkspJdS) N param_backend*rh)rMracsrbr;r!rrdr+r+r.rfrz=test_parameterized_backend_context_manager..rcSr}r+rrdr+r+r.r/rz>test_parameterized_backend_context_manager..)rIr&rLr_r?r#r r^r]rMr rarrj)rrFrGrrr+r+r.*test_parameterized_backend_context_managers    rQcCsttksJttdddd?t\}}t|tksJ|jdks&J|dks,Jt}|j dks6J|j |us=J|ddt dD}Wdn1sRwY|ddt dDksdJttkskJdS) N+)rMrrrcsrbr;rPrdr+r+r.rfrzFtest_directly_parameterized_backend_context_manager..cSr}r+rrdr+r+r.r/rzGtest_directly_parameterized_backend_context_manager..) r_r?r#rLr r^r]rMr rarrj)rFrGrrr+r+r.3test_directly_parameterized_backend_context_managers   rScCstdtS)Nr)rosgetpidr+r+r+r.sleep_and_return_pidsrVcCs<ttksJtdksJtddddtdDS)NrrBrrcs|]}ttVqdSr;)r!rVrr+r+r.rfsz"get_nested_pids..)r_rr _effective_n_jobsrjr+r+r+r.get_nested_pidss rYcs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSr\)superrZget_nested_backendr __class__r+r.r\szMyBackend.get_nested_backend)rOrPrQrRr\ __classcell__r+r+r]r.rZsrZback_compat_backend)r2r1r0r`cCsht|&tddddtdD}|D] }tt|dks!JqWddS1s-wYdS)NrBrrcsrWr;)r!rYrr+r+r.rfs  z6test_nested_backend_context_manager..rr)r#r rjrset)r9 pid_groups pid_groupr+r+r.#test_nested_backend_context_managers "rd)rBrqNcs~ddtddfddtdDtdtddfddtdDWddS1s8wYdS) NcSs0tt|ks Jt|}t|ksJdSr;)r_r&r$r rX)expected_backend_typeexpected_n_jobr+r+r.check_nested_backend sz?test_nested_backend_in_sequential..check_nested_backendrrrc3s|] }tddVqdS)r1rNr!r)rgr+r.rf  z4test_nested_backend_in_sequential..rc3s|] }tVqdSr;rhrr9rgrar+r.rfri)r rjr#)r9rar+rjr.!test_nested_backend_in_sequentials  "rkcCs@t|\}}|j|ksJWddS1swYdSr;)r# nesting_level) inner_backendexpected_levelr9rar+r+r.check_nesting_level"s"ro outer_backendrmcszt|dtd|dfddtdDtddtfddtdDWddS1s6wYdS) NrrBr`c3rrr!rorrmr+r.rf.riz-test_backend_nesting_level..rrrc3rrrqrrrr+r.rf4s)ror rjr#)rprmr+rrr.test_backend_nesting_level's  "rscsddlGfdddt}td|ddtd"\}}tddfd d td D|jd ks5JWddS1s@wYdS) NrcseZdZdZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrsr rr+r+r.retrieval_context?s z;test_retrieval_context..MyBackend.retrieval_contextN)rOrPrQrecontextmanagerrtr+) contextlibr+r.rZ<srZ retrievalcSstddddt|DS)NrBrrcsrbr;rrdr+r+r.rfGrz>test_retrieval_context..nested_call..ri)nr+r+r. nested_callFsz+test_retrieval_context..nested_callrBrrc3s|] }t|VqdSr;rhrd)ryr+r.rfJs  z)test_retrieval_context..rr)rvrr"r#r rjre)rZbarr+)rvryr.test_retrieval_context8s "r{cCstd}t|t|dS)Nfoobar)r)reprpickledumps)er+r+r.test_joblib_exceptionSsrcCsxtt}tt |ddWdn1swYtt}tt |dWddS1s5wYdS)Nrrr<)rr>rrrYr*) safe_divisionsafe_interruptr+r+r.test_safe_function\s    "rr)rrqgQ?cCs8ttt|dWddS1swYdS)Nr)rrTr rr+r+r.test_invalid_batch_sizefs  "rz)n_tasks, n_jobs, pre_dispatch, batch_size) )rBrBrr)rBrBrarrrBrar)irBrarr)rrrar)rrar)r 2 * n_jobsr)rrr)rrrrS)rrrrcCsN|||d}ddt|D}tdi|ddt|D}||ks%JdS)N)rarrcSr}r+r~rdr+r+r.r/~rz0test_dispatch_race_condition..csrbr;rcrdr+r+r.rfrz/test_dispatch_race_condition..r+r)n_tasksrarrparamsrrr+r+r.test_dispatch_race_conditionls  rcCs8t}tddd}|jd}|}||ksJdS)NrBr0r`context)r get_start_methodr _backend_argsget)mp_start_methodrr start_methodr+r+r.test_default_mp_contexts   rcs^|dkr td}tjd}|ddtjtd|dfddt dDdS) Nr0r4rOirBr`c3s"|] }ttjjVqdSr;)r!rdotTrdrHr+r.rfrzAtest_no_blas_crash_or_freeze_with_subprocesses..) r r3rrandom RandomStaterandnrrr rj)r9rngr+rr..test_no_blas_crash_or_freeze_with_subprocessess    rafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr0forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) r rpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrsys executable)r9coder+r+r.2test_parallel_with_interactively_defined_functionss    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r!rkwargsc Csv|dvr|tks tjdkrtdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r0r4win32zNot picklable with pickle)rr9rjoblib_root_folderzunpicklable_func_script.pyrrr) SQUARE_MAINrplatformrr)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrrTpathdirnamejoblib__file__rwriterrstrpath)r9rrtmpdirr code_filer+r+r.0test_parallel_with_unpicklable_functions_in_argss     raimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyr)rr)rr5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrrrscriptr+r+r.Btest_parallel_with_interactively_defined_functions_default_backend4s    rauimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\] LokyProcess)r stderr_regexr)rr9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrrrr+r+r.5test_parallel_with_interactively_defined_bound_method]s    rcCs"tg}tdd|gksJdS)NrBrr)iterr )exhausted_iteratorr+r+r.%test_parallel_with_exhausted_iteratorgsrcCs"t|tjs tdt||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorr]copyrr+r+r. check_memmapls rcCsdd}tdd|ddd|dD}t||t|D] \}}tj||qtd d|dd d|dD}t||t|D] \}}tj||qBdS) Ncss*t|D] }tjdtjd|VqdS)Nrdtype)rjronesfloat32)rxrer+r+r.generate_arrays{s zBtest_auto_memmap_on_arrays_from_generator..generate_arraysrBrra max_nbytesr9csrbr;r!rr,rHr+r+r.rfrgz.r|rcsrbr;rrr+r+r.rfrg)r rrrtestingassert_array_equal)r9rrresultrr+r+r.)test_auto_memmap_on_arrays_from_generatorssrcCs|Sr;r+)argr+r+r.identityrMrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj s;Jd j |ksDJtj ||dS) Nz test.mmapuint8rr) mmap_moderBrrc3rr;)r!rrrr+r.rfrz.test_memmap_with_big_offset..)rr)rrmmapALLOCATIONGRANULARITYrzerosrr r r rroffsetrr)rfnamesizeobjrr+rr.test_memmap_with_big_offsets   rcCstjdd}tddddtdDWdn1swYt|dks,J|d}t|jts8Jt|jd ksAJdS) NTrr)rcsrbr;rcrdr+r+r.rfrzFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr rjrrrrAr)rrr+r+r.3test_warning_about_timeout_not_supported_by_backends r)rrBryrqcCsvdgdgd}ttt}t||ddd|DWdn1s(wYt|}|dks9JdS)NrHrr|r`css|] }ttj|VqdSr;r!rWrrdr+r+r.rfrz%test_abort_backend..)rrrWr )rar9delayst_startdtr+r+r.test_abort_backends   rcCs|j}tdd||d#}|ddtjdgdDtt|dks'JWdn1s1wYtdD] }t|sCn td q:t d tdd|d }|d dtjdgdDtdD]}t|sqdStd qgt d ) NrBr)rarr9 temp_foldercsrbr;rrr+r+r.rfrz(test_memmapping_leaks..rrr|rz/temporary directory of Parallel was not removedrcsrbr;rrr+r+r.rfr) rr rrrrTlistdirrjrAssertionError)r9rrrr+r+r.test_memmapping_leakss(       r)Nr1r2cCs<td|dddtdD}|ddtdDksJdS)NrBr`css |] }tdd|VqdS)cSr@rAr+rCr+r+r.sz2test_lambda_expression...Nrhrdr+r+r.rfs z)test_lambda_expression..rcSsg|]}|dqS)rBr+rdr+r+r.r/rz*test_lambda_expression..ri)r9rr+r+r.test_lambda_expressionsrcsd}d}d|td||d}|fddt|D|jj|jjks&J|jj|jjks0J|fddt|D|jj|jjksGJ|jj|jjksQJd S) zCTest that a parallel backend correctly resets its batch statistics.rBig@r)rzrar9c3|] }ttjVqdSr;rrd task_timer+r.rfrz6test_backend_batch_statistics_reset..c3rr;rrdrr+r.rfrN)r rjr_effective_batch_size_DEFAULT_EFFECTIVE_BATCH_SIZE_smoothed_batch_duration _DEFAULT_SMOOTHED_BATCH_DURATION)r9ran_inputsrr+rr.#test_backend_batch_statistics_resets$ rcCsdD];}tt|djtksJt|dd}t|jtksJt|dd}t|jtks.Jt|dd}t|jtks=Jqtdd dd }t|jtksNJtd ddtdd }t|jtkscJ|jdksjJWdn1stwYtd ddtd dd}t|jtksJ|jd ksJWdn1swYtd ddtdd }t|jtksJ|jdksJWdn1swYtd ddtd dd}t|jtksJ|jd ksJWddS1swYdS)N)rrBrqrrthreadsraprefer processes sharedmemrarequirerBr1)rar9rrrhrr)r]r rrrr#ra)rarr+r+r.$test_backend_hinting_and_constraintss<       "rcCszGdddt}t|&tddd}t|j|ksJtddd}t|j|ks,JWdn1s6wYGdd d t}t|Otddd}t|j|ksXJ|\}}|d ksdJ|d ksjJtddd d }t|jtkszJ|\}}d }||ksJ|d ksJWdn1swYtt t|ddWddS1swYdS)Nc@$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSrFr;r+rr+r+r. apply_async5rMzgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccS|Sr;r+r8r+r+r.r$8rMzltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrOrPrQsupports_sharedmem use_threadsrr$r+r+r+r.MyCustomThreadingBackend1  r rBrrrrc@r)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSrFr;r+rr+r+r.rFrMzhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSr r;r+r8r+r+r.r$IrMzmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNr r+r+r+r.MyCustomProcessingBackendBrrr.r)rarrzzUsing ThreadingBackend as joblib.Parallel backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r9r) rr#r r]r readouterrrstriprrT)capsysr rrouterrrr+r+r.9test_backend_hinting_and_constraints_with_custom_backends.s0        "rcCstt tddWdn1swYtt tddWdn1s-wYtttdddWdn1sGwYtdurtttdddWdn1sewYtttd ddWddS1swYdSdS) Ninvalidrrrr)rrr1rr0)rrTr r r+r+r+r.,test_invalid_backend_hinting_and_constraintsbs"       "rcCsFtj}td}ttjtsJ|ttjt|us!JdS)Nr2)r rr#rr unregisterr])defaultpbr+r+r.test_global_parallel_backendws rcCsPdd}|td<tdttjtsJWddS1s!wYdS)NcSs ttd<dS)Nfoo)rr&r+r+r+r. register_foo z,test_external_backends..register_foor)r(r#rr rr)rr+r+r.test_external_backendss  "r c stdd4}t|jj|jjfg}dkr|WdS|fddtdD}||dWdS1s.r)r r]rrOrlrj)r!rr this_levelrr+r#r.r"s  $r"cCst|dd t}Wdn1swYtdkr#d}d}nd}d}|d}|df|dfd|fd|fg}||ksAJdS)NrBrrrrrBackendr)r#r"r%title)r9backend_types_and_levelssecond_level_backend_type max_leveltop_level_backend_typeexpected_types_and_levelsr+r+r.test_nested_parallelism_limits  r,zThis test requires daskc Cstjddd}tjtdtjd}tdD].}td t|d}Wdn1s+wYt |dks8Jt dd |DsCJqtd t}Wdn1sVwYt |dkscJt d d |DsnJdS) NrB) n_workersthreads_per_workergcArdask)datarcs|] \}}|dkVqdSDaskDistributedBackendNr+r,rrr+r+r.rfz4test_nested_parallelism_with_dask..csr1r2r+r4r+r+r.rfr5) distributedClientrrintrrjr#r"rr)clientr0rer'r+r+r.!test_nested_parallelism_with_dasks"    r:cCstddtdDS)z6A horrible function that does recursive parallel callscsrWr;)r!_recursive_parallelrdr+r+r.rfsz&_recursive_parallel..rBri) nesting_limitr+r+r.r;sr;c Cst|ddtt }tWdn1swYWdn1s(wY|j}|dkrHddlm}t||rFt ddSdSt|t sOJdS)NrBrrr1r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r#r BaseExceptionr;r&joblib.externals.loky.process_executorr=rrxfailRecursionError)r9r0excr=r+r+r.test_thread_bomb_mitigations    rCcCs*i}dD] }tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSVECLIB_MAXIMUM_THREADSNUMEXPR_NUM_THREADSNUMBA_NUM_THREADS ENABLE_IPCr|)rTenvironrr)env_varsvarr+r+r._run_parallel_sums rNzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]-\}}||ks#J|D]\}}|dr9|t|ks8Jq'|dks?J|dksEJq'qdS) NrBr`csrWr;)r!rNrr+r+r.rf  z-test_parallel_thread_limit..r_THREADSrJ1)r rjr7r%itemsendswithr)r9rexpected_num_threadsworker_env_varsomp_num_threadsrrr+r+r.test_parallel_thread_limits    rWz%This test requires dask NOT installedcCs:ttdd tdWddS1swYdS)NzPlease install daskrr/)rrTr#r+r+r+r.)test_dask_backend_when_dask_not_installeds "rXc CsGdddt}d}t|6tjt|dtddddtdDWdn1s0wYWddSWddS1sHwYdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_dSr\r+)rGrrr+r+r.r6rMz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSr)NzNo worker availabler)rGfunccallbackr+r+r.r r?z?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSrYr\r+r8r+r+r.r$rMzDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsr;)rOrPrQr6rr$r+r+r+r.ZeroWorkerBackends  r\z&ZeroWorkerBackend has no active workerrrBrrcsrbr;rrdr+r+r.rfrz+test_zero_worker_backend..)rr#rrrr rj)r\ expected_msgr+r+r.test_zero_worker_backends "r^csdadddks JtddfddtdD}t|dhks&Jdadks/Jtddfd dtdD}t|dhksHJdS) Nzoriginal valuecSstSr;)MY_GLOBAL_VARIABLEr+r+r+r. check_globalssz@test_globals_update_at_each_parallel_call..check_globalsrBrrc3|]}tVqdSr;rhrdr`r+r.rf%rOz.z changed valuec3rar;rhrdrbr+r.rf.rO)r_r rjra)workers_global_variabler+rbr.)test_globals_update_at_each_parallel_callsrdcCs4ddl}|jdd}|||ddlm}|S)Nrr|)threadpool_info)numpyrrr threadpoolctlre)rrHrer+r+r._check_numpy_threadpool_limits8s   rhcCs4|D]}|d|dkr|dSqtd|)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rTr) child_module parent_info parent_moduler+r+r._parent_max_num_threads_forCs rncCs@|D]}|D]}t||}t|||h}|d|vsJqqdS)Nrj)rnmin) workers_inforlrjchild_threadpool_inforkparent_max_num_threadsrr+r+r.check_child_num_threadsKsrs)rBrryrqcCsbt}t|dkrtjddt|dddtdD}t|}tt|d}t |||dS) Nr&Need a version of numpy linked to BLASmsgrrcsrWr;r!rhrdr+r+r.rfcrOz6test_threadpool_limitation_in_child..rBr) rhrrrr rjr$r7r%rs)rarlworkers_threadpool_infosexpected_child_num_threadsr+r+r.#test_threadpool_limitation_in_childWs  rzinner_max_num_threads)rrBrNrBcCst}t|dkrtjddtd|dt|dddtd D}Wdn1s.wYt|}|durDtt |d }n|}t |||dS) Nrrtrur1r{rrcsrWr;rwrdr+r+r.rf{rOz>test_threadpool_limitation_in_child_context..rBr) rhrrrr#r rjr$r7r%rs)rar{rlrxryr+r+r.+test_threadpool_limitation_in_child_contextms   r}var_name)rErFrDc stddddtj}z_dtj<t|dfddtd D}|ddgks0Jtd d d t|dfd dtd D}Wdn1sRwY|ddgks_JW|durjtj=dS|tj<dS|durztj=w|tj<w)NTreusecSs tj|Sr;)rTrKr)r~r+r+r._get_envrz>test_threadpool_limitation_in_child_override.._get_env4rrc3|] }tVqdSr;rhrdrr~r+r.rfrgz?test_threadpool_limitation_in_child_override..rBr1rr|c3rr;rhrdrr+r.rfrgrQ)rshutdownrTrKrr rjr#)rar~original_var_valuerr+rr.,test_threadpool_limitation_in_child_overrides*     rcCs>ttddt|ddWddS1swYdS)Nz#does not acc.*inner_max_num_threadsrrr|)rrr#r8r+r+r.1test_threadpool_limitation_in_child_context_errors"r)rBrrqcCsJdd}||tdd}tdD]}||tdd}||ks"JqdS)Ncs,tdt|dfddtdDdS)Nrrrc3rr;)r!sumrdrCr+r.rfrzAtest_loky_reuse_workers..parallel_call..rrrr+rCr. parallel_calls$z.test_loky_reuse_workers..parallel_callTrr)rrj)rarfirst_executorrexecutorr+r+r.test_loky_reuse_workerss   r)F)rr)rhr;)rRrTrrWrrr2 tracebackrmathrrr~rr0rrrr r r joblib._multiprocessing_helpersr joblib.test.commonrrrjoblib.testingrrrrrjoblib.externals.lokyrrrposix ImportError _openmp_test_helper.parallel_sumrr6joblib._parallel_backendsrrrrrrjoblib.parallelr r!r"r#r$r%r&r'r(joblib.my_exceptionsr)r*sortedkeysALL_VALID_BACKENDSPROCESS_BACKENDSPARALLEL_BACKENDShasattrrr3r?r:r>rDrrErVrYr[r_rlrnrpmarkrxrrrrrrrrrrrrrrrrrrrr#r$r&r*r,r@r3r4r9r;r=r@rBrHrJextendrjrKrLrQrSrVrY_parallel_backendsrZrdrkrorsr{rrrrrrrrrr SQUARE_LOCAL SQUARE_LAMBDArrrrrrrrrrrrrrrrrrrrrrrr r"r,r:r;rCrNrWrXr^rdrhrnrsrzr}rrrr+r+r+r.sJ                                             I                          ( ,         #   .4         !