iordZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z ddl mZmZmZmZmZddlmZ e ddlZddlmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&d Z'e(dZ)ejTejVzejXzZ-ejTejVzZ.e/Z0dZ1dZ2dZ3e3e"jhd<GddZ5dZ6d dZ7dZ8dZ9dZ:dZ;Gdde<Z= d!dZ>Gdde<Z?y#e$r edZYwxYw#e$rdZYwxYw)"z/ Reducer using memory mapping for numpy arrays N)mmap)util)HIGHEST_PROTOCOL PicklingErrordumpsloads whichmodule)uuid4) as_strided) make_memmap) delete_folder)resource_tracker)dumploadload_temporary_memmapz/dev/shmgeAcddlm}tjdj t j j|t j|j|dy)Nr )_resource_trackerzj[FINALIZER CALL] object mapping to {} about to be deleted, decrementing the refcount of the file (pid: {})file) 'externals.loky.backend.resource_trackerrrdebugformatospathbasenamegetpid maybe_unlink)filenamers o/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/joblib/_memmapping_reducer.py_log_and_unlinkr ;sLJJJ ;;A6 GG  X & <  ""8V4c 2tjdjt|t |t j j|jt jtj|t|jy)NzD[FINALIZER ADD] adding finalizer to {} (id {}, filename {}, pid {})) rrrtypeidrrrrrweakreffinalizer )memmaps radd_maybe_unlink_finalizerr(Gs^JJNUU L"V*bgg&6&6v&G   V_foo>r!cd}td|dzD]} tj|yy#t$rBt j dj |||k(rtjdYgt$rYqwxYw)a=Wrapper around os.unlink with a retry mechanism. The retry mechanism has been implemented primarily to overcome a race condition happening during the finalizer of a np.memmap: when a process holding the last reference to a mmap-backed np.memmap/np.array is about to delete this array (and close the reference), it sends a maybe_unlink request to the resource_tracker. This request can be processed faster than it takes for the last reference of the memmap to be closed, yielding (on Windows) a PermissionError in the resource_tracker loop. r z9[ResourceTracker] tried to unlink {}, got PermissionErrorg?N) rangerunlinkPermissionErrorrrrtimesleepFileNotFoundError)r NUM_RETRIESretry_nos r unlink_filer3PsK![1_-  IIh  . JJKRR  ;& 3    s/AB9BBrc(eZdZdZdZdZdZdZy)_WeakArrayKeyMapaA variant of weakref.WeakKeyDictionary for unhashable numpy arrays. This datastructure will be used with numpy arrays as obj keys, therefore we do not use the __get__ / __set__ methods to avoid any conflict with the numpy fancy indexing syntax. ci|_yN_dataselfs r__init__z_WeakArrayKeyMap.__init__{s  r!cd|jt|\}}||ur t||Sr7)r9r$KeyError)r;objrefvals rgetz_WeakArrayKeyMap.get~s3::bg&S 5 3-  r!ct| j\}}||ur t| ||fj<y#t$rfd}tj||}Y9wxYw)Nc j=yr7r8)_keyr;s r on_destroyz(_WeakArrayKeyMap.set..on_destroysJJsOr!)r$r9r>r%r@)r;r?valuer@rErGrFs` @rsetz_WeakArrayKeyMap.setsxg /ZZ_FCuCsm# u* 3 / $++c:.C /s&A%A0/A0ctd)Nz"_WeakArrayKeyMap is not pickleable)rr:s r __getstate__z_WeakArrayKeyMap.__getstate__s@AAr!N)__name__ __module__ __qualname____doc__r<rBrIrKr!rr5r5ss%$Br!r5c\t|dd}|yt|tr|St|S)z@Recursively look up the original np.memmap instance base if any.baseN)getattr isinstancer_get_backing_memmap)abs rrUrUs664 Ay At #1%%r!cd}| tjjdd}|tjj t rt tdr tjt }|j|jz}|tkDr\t }tjj||}tjj |stj|d}|tj }tjj#tjj%|}tjj||}||fS#ttf$rd}YwxYw)aGet the full path to a subfolder inside the temporary folder. Parameters ---------- pool_folder_name : str Sub-folder name used for the serialization of a pool instance. temp_folder: str, optional Folder to be used by the pool for memmapping large arrays for sharing memory with worker processes. If None, this will try in order: - a folder pointed by the JOBLIB_TEMP_FOLDER environment variable, - /dev/shm if the folder exists and is writable: this is a RAMdisk filesystem available by default on modern Linux distributions, - the default system temporary folder that can be overridden with TMP, TMPDIR or TEMP environment variables, typically /tmp under Unix operating systems. Returns ------- pool_folder : str full path to the temporary folder use_shared_mem : bool whether the temporary folder is written to the system shared memory folder or some other temporary folder. FNJOBLIB_TEMP_FOLDERstatvfsT)renvironrBrexistsSYSTEM_SHARED_MEM_FShasattrrZf_bsizef_bavailSYSTEM_SHARED_MEM_FS_MIN_SIZEjoinmakedirsIOErrorOSErrortempfile gettempdirabspath expanduser)pool_folder_name temp_folderuse_shared_mem shm_statsavailable_nbytes pool_folders r _get_temp_dirrps)<Njjnn%94@  77>>. /GB 4J #JJ';< #,#4#4y7I7I#I #&CC#7K"$'',,{>+6 K0%)N ))+ ''//"''"4"4["ABK'',,{,<=K  &&W% ##  #sBE((E<;E<ct|duS)z?Return True if a is backed by some mmap buffer directly or not.N)rU)rVs rhas_shareable_memoryrrs q ! --r!c x|dk(rd}|t|||||||St|||||||} t| ||S)z2Reconstruct an array view on a memory mapped file.zw+zr+)dtypeshapemodeoffsetorderunlink_on_gc_collect)rtrurwrvrxry)rustrides)r r ) rrtrvrwrxrurztotal_buffer_lenryrRs r_strided_from_memmapr|si t| !5   "!5 $eW==r!c Vtjdj|jt j  ddlm}||\}}||d}||z }||jz }|jdkDr|jdrd}nd}|jds|jdrd }d } n|j}||z |jz} t |j"|j$|j&|||j|| d f fS#ttf$r ddl m}YwxYw) aPickling reduction for memmap backed arrays. a is expected to be an instance of np.ndarray (or np.memmap) m is expected to be an instance of np.memmap on the top of the ``base`` attribute ancestry of a. ``m.base`` should be the real python mmap object. zC[MEMMAP REDUCE] reducing a memmap-backed array (shape, {}, pid: {})r) byte_boundsr F_CONTIGUOUSFC C_CONTIGUOUSNF)rrrrurrnumpy.lib.array_utilsr~ModuleNotFoundError ImportErrornumpyrwndimflagsrzitemsizer|rrtrv) rVmr~a_starta_endm_startrwrxrzr{s r_reduce_memmap_backedrs( JJMTT GGRYY[  &5!^NGU!nQG w F ahhF vvzaggn-ww~!''."9))!GO :  JJ GG FF   GG      ;  -&%&sDD('D(ct|}t|tjr|jt vr t ||Stttj|tffS)z5reduce a np.array or a np.memmap from a child processprotocol) rUrTnpr'r JOBLIB_MMAPSrrrasarrayr)rVrs rreduce_array_memmap_backwardrWsTAA!RYYAJJl$B%Q**bjjm6FGIJJr!c8eZdZdZ ddZedZdZdZy)ArrayMemmapForwardReducera#Reducer callable to dump large arrays to memmap files. Parameters ---------- max_nbytes: int Threshold to trigger memmapping of large arrays to files created a folder. temp_folder_resolver: callable An callable in charge of resolving a temporary folder name where files for backing memmapped arrays are created. mmap_mode: 'r', 'r+' or 'c' Mode for the created memmap datastructure. See the documentation of numpy.memmap for more details. Note: 'w+' is coerced to 'r+' automatically to avoid zeroing the data on unpickling. verbose: int, optional, 0 by default If verbose > 0, memmap creations are logged. If verbose > 1, both memmap creations, reuse and array pickling are logged. prewarm: bool, optional, False by default. Force a read on newly memmapped array to make sure that OS pre-cache it memory. This can be useful to avoid concurrent disk access when the same data array is passed to different worker processes. c ||_||_||_t||_|dk(r&|j j t |_n||_||_t|_ t|_ ||_ y)Nauto) _max_nbytes_temp_folder_resolver _mmap_modeintverbose _temp_folder startswithr]_prewarmr5_memmaped_arraysrI_temporary_memmaped_filenames_unlink_on_gc_collect)r; max_nbytestemp_folder_resolver mmap_moderyrprewarms rr<z"ArrayMemmapForwardReducer.__init__su&%9"#7| f  $ 1 1 < <=Q RRDM#DM 0 2-0U*%9"r!c"|jSr7)rr:s rrz&ArrayMemmapForwardReducer._temp_folders))++r!c|jd|j|jf}|j|jd}t ||fS)N)rr)rrrrrr)r;argskwargss r __reduce__z$ArrayMemmapForwardReducer.__reduce__sF  $9S9ST||}} )$66r!c  t|}|&t|tjr t ||S|j j s@|j3|j|jkDr tj|jtj|jt |j"j%|}tj8j;|j|}||j<v}|j<j?||j@rtCjD|d|rtCjD|dtj8jG|stIjJdj)|jL|j |tO||D]}tj|tP|jRr}tU||jVjYnWtIjJdj)|jL|j tj8j[|t\||jV|j@ffStIjJdj)|jL|j t^ta|tbffS#t$r*}|jtj k7r|Yd}~Td}~wwxYw#t&$rrdj)tj*t-t/j0t3j4}|j"j7||YwxYw)Nz {}-{}-{}.pklrzP[ARRAY DUMP] Pickling new array (shape={}, dtype={}) creating a new memmap at {})rzN[ARRAY DUMP] Pickling known array (shape={}, dtype={}) reusing memmap file: {}zB[ARRAY DUMP] Pickling array (NO MEMMAPPING) (shape={}, dtype={}).r)2rUrTrr'rrt hasobjectrnbytesrrcrchmodFOLDER_PERMISSIONSreerrnoEEXISTrrBr>rrr$ threadingcurrent_threadr hexrIrrbraddrrregisterr\rrrurFILE_PERMISSIONSrrrmaxrrrrr)r;rVrerr is_new_memmapdumped_filenames r__call__z"ArrayMemmapForwardReducer.__call__s  " =Z2995(A. .!!  ,4+++  D--.**,>? 70044Q7ww||D$5$5x@H %D,N,NNM  . . 2 28 <))!))(F; !))(F;77>>(+ 228&!''82T(,Ax'8OHH_.>?(9== T__=AAC ..4f"''*:*:8*D/&4??D,F,FG  JJ$fQWWagg6 E!.>?AB Bi 77ell*G+   7 *00IIKI$<$<$>!?%%))!X6 7s,3AK7L L%L  LA7N  N N)rT) rLrMrNrOr<propertyrrrrPr!rrrhs4<:,,,7"dCr!rc | t}| t}tet||||||} | |tj<| |tj<t |tj<t |tj<||fS)a Construct a pair of memmapping reducer linked to a tmpdir. This function manage the creation and the clean up of the temporary folders underlying the memory maps and should be use to get the reducers necessary to construct joblib pool or executor. )r)dictrrndarrayr'r) forward_reducersbackward_reducersrrrrrryrforward_reduce_ndarrays rget_memmapping_reducersrs"6  F ~";    " (>$&<# )E"**%'C"))$ . ..r!c:eZdZdZd dZdZdZdZdZ d dZ y) TemporaryResourcesManagera9Stateful object able to manage temporary folder and pickles It exposes: - a per-context folder name resolving API that memmap-based reducers will rely on to know where to pickle the temporary memmaps - a temporary file/folder management API that internally uses the resource_tracker. Ncd|_||_d|_t|_t j |_i|_|t j }|j|yr7) _current_temp_folder_temp_folder_root_use_shared_memr_cached_temp_foldersr r_id _finalizersset_current_context)r;temp_folder_root context_ids rr<z"TemporaryResourcesManager.__init__LsZ$(!!1#$(F!7;;   J   ,r!c4||_|j|yr7)_current_context_idregister_new_context)r;rs rrz-TemporaryResourcesManager.set_current_context[s#-  !!*-r!c||jvrydjtj|j|}t ||j \}}|j||||j|<y)Nz!joblib_memmapping_folder_{}_{}_{})rrrrrrprregister_folder_finalizer)r;rnew_folder_namenew_folder_pathrEs rrz.TemporaryResourcesManager.register_new_context_sr 22 2 BHH TXXzO"/@V@V!W OQ  * *?J G4CD % %j 1r!c4|j|jS)z@Return a folder name specific to the currently activated context)rrr:s rresolve_temp_folder_namez2TemporaryResourcesManager.resolve_temp_folder_namevs(()A)ABBr!cttdtjdfd}t j||j |<y)Nrfolderctdgj} |dtjdy#t$r't j djYywxYw)Nr)fromlistTallow_non_emptyrz%Failed to delete temporary folder: {}) __import__rr unregisterrewarningswarnr)rpool_module_namepool_subfolders r_cleanupzETemporaryResourcesManager.register_folder_finalizer.._cleanupsg' O+<m  ndC ++NHE  ;BB>R s <-A,+A,)r rrratexitr)r;rrrrs ` @rrz3TemporaryResourcesManager.register_folder_finalizer|sB '}oF!!.(; ((.x'@$r!c|/t|jD]}|j|||y|jj|}|rtj j |rt j|D]m}|r5tjtj j||d:tjtj j||do||z} t|||jj|dtj|d|jj|d}|tj|yyyy#t $rYywxYw)z9Clean temporary resources created by a process-based poolN)forcerrrr)listr_clean_temporary_resourcesrBrrr\listdirrrrbrrpoprrre)r;rrrrkr finalizers rrz4TemporaryResourcesManager._clean_temporary_resourcessT  #4#<#<= //e_0> 3377 CKrww~~k: " ; 7H)33GGLLh?)55GGLLh?!8 5(!+O--11*dC$// XF!% 0 0 4 4Z FI ,)))4-9 ;{> s:A2E00 E<;E<)NN)NFF) rLrMrNrOr<rrrrrrPr!rrrBs/ -.D.C A@=B0r!rr7)NNNg.ArrFT)@rOrrrstatrfrr.rr%rmultiprocessingrpicklerrrrr uuidr WindowsError NameErrorr#rrnumpy.lib.stride_tricksr r backportsr diskrexternals.loky.backendr numpy_picklerrrr]rraS_IRUSRS_IWUSRS_IXUSRrrrIrr r(r3_CLEANUP_FUNCSr5rUrprrr|rrobjectrrrrPr!rrsi   MM2#4;;"!$C \\DLL04<<?<<$,,. u  5?@+6'&B&BZ&"8'v. '>T9xK"hChCX ,/^IIU:L  Bs$D D, D)(D),D65D6