+
    QV-j€_  ã                   óâ   € R t ^ RIt^ RIt^ RIHt ^ RIHt ^ RIHt ^ RI	t
^ RIt^ RIHt  ^ RIHt ^RIHt R tR	 tR
 tRR ltR t ! R R4      t ! R R4      tR tR#   ] d    Rt L;i ; i)zTF-specific utils import.N)Úpartial)Úceil)Úuuid4)Úget_context)ÚSharedMemory)Úconfigc                 óp  € \        V \        4      '       d   V # \        P                  '       d   ^ RIpM\        R4      hV ^ ,          p/ pVP                  4        FÉ  w  rE\        V\        P                  4      '       d1   \        P                  ! V  Uu. uF  qfV,          NK  	  up4      W4&   KU  \        WQP                  4      '       d,   TP                  V  Uu. uF  qfV,          NK  	  up4      W4&   K›  \        P                  ! V  Uu. uF  qfV,          NK  	  up4      W4&   KË  	  V# u upi u upi u upi )é    NúFCalled a Tensorflow-specific function but Tensorflow is not installed.)Ú
isinstanceÚdictr   ÚTF_AVAILABLEÚ
tensorflowÚImportErrorÚitemsÚnpÚndarrayÚstackÚTensorÚarray)ÚfeaturesÚtfÚfirstÚbatchÚkÚvÚfs   &      Úh/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/datasets/utils/tf_utils.pyÚminimal_tf_collate_fnr   $   sê   € Ü�(œD×!Ò!ØˆÜ	×	×	Ð	ÜäÐbÓcÐcà�Q�K€EØ€EØ—‘–‰ˆÜ�aœŸ™×$Ò$Ü—x’x©xÓ 8©x¨! 1§ ©xÑ 8Ó9ˆE‹HÜ˜Ÿ9™9×%Ò%Ø—x‘x©xÓ 8©x¨! 1§ ©xÑ 8Ó9ˆE‹Hä—x’x©xÓ 8©x¨! 1§ ©xÑ 8Ó9ˆE‹Hñ ð €Lùò !9ùâ 8ùâ 8s   ÂD)
ÃD.
Ä	D3
c                 óH   € \        V 4      pR V9   d   VR ,          VR&   VR  V# )ÚlabelÚlabels)r   )r   r   s   & r   Ú#minimal_tf_collate_fn_with_renamingr"   8   s-   € Ü! (Ó+€EØ�%ÔØ �.ˆˆh‰Ø�'ˆNØ€Ló    c                 óP  € \         P                  P                  V 4      '       d   \        V P                  4      # \         P                  P                  V 4      ;'       gG    \         P                  P                  V 4      ;'       g     \         P                  P                  V 4      # ©N)ÚpaÚtypesÚis_listÚis_numeric_pa_typeÚ
value_typeÚ
is_integerÚis_floatingÚ
is_decimal)Úpa_types   &r   r)   r)   @   sl   € Ü	‡x�x×Ñ˜× Ò Ü! '×"4Ñ"4Ó5Ð5Ü�8‰8×Ñ˜wÓ'×hÐh¬2¯8©8×+?Ñ+?ÀÓ+H×hÐhÌBÏHÉH×L_ÑL_Ð`gÓLhÐhr#   c                 óæ  € \        V \        P                  4      '       g   V P                  4       p R p\        V \        P                  4      '       d   WP                  4       ,          pRpM�\        P                  ! \        P                  ! V 4      ^8H  4      '       d   W^ ,          V R,          ^,            pM@\        V \        P                  4      '       d
   W,          pM\        R\        V 4       24      hVe3   VP                  4        U	U
u/ uF  w  ršW’9   g
   V	R9   g   K  WšbK  	  pp	p
V'       dp   \        \        VP                  4       4      ^ ,          4      p\        V4       UU	U
u. uF-  qÈP                  4        U	U
u/ uF  w  ršWšV,          bK  	  up
p	NK/  	  pp	pp
V! V3/ VB pV'       dO   / pVP                  4        F6  w  rï\        P                  ! WŽ,          4      pVP!                  V4      pVWÞ&   K8  	  V# . pVP                  4        FC  w  rï\        P                  ! WŽ,          4      pVP!                  V4      pVP#                  V4       KE  	  V# u up
p	i u up
p	i u up
p	pi )TFzUnexpected type for indices: éÿÿÿÿ)r    Ú	label_idsr!   )r   r   r   ÚnumpyÚintegerÚitemÚallÚdiffÚRuntimeErrorÚtyper   ÚlenÚlistÚvaluesÚranger   ÚastypeÚappend)ÚindicesÚdatasetÚcols_to_retainÚ
collate_fnÚcollate_fn_argsÚcolumns_to_np_typesÚreturn_dictÚ
is_batchedr   ÚkeyÚvalueÚactual_sizeÚiÚ	out_batchÚcolÚ
cast_dtyper   s   &&&&&&&          r   Únp_get_batchrN   F   sû  € ô �gœrŸz™z×*Ò*Ø—-‘-“/ˆà€Jä�'œ2Ÿ:™:×&Ò&ØŸ™›Õ'ˆØ‰
Ü	�Š”—’˜Ó  AÑ%×	&Ò	&Ø �
 W¨R¥[°1¥_Ð5‰Ü	�GœRŸZ™Z×	(Ò	(ØÕ ‰äÐ:¼4À»=¸/ÐJÓKÐKàÒ!ð $Ÿk™kœmô
á+‘
�ØÔ$¨Ð/OÑ(Oô ˆCŠJÙ+ð 	ñ 
÷ Üœ$˜uŸ|™|›~Ó.¨qÕ1Ó2ˆäJOÐP[ÔJ\Õ]ÑJ\ÀQ·+±+´-Ô@±-¡J C�#˜Q•x’-±-Õ@ÑJ\ˆÒ]Ù�uÑ0 Ñ0€EçØˆ	Ø2×8Ñ8Ö:‰OˆCä—H’H˜U�ZÓ(ˆEØ—L‘L Ó,ˆEØ"ˆI‹Nñ	  ;ð Ðð ˆ	Ø2×8Ñ8Ö:‰OˆCä—H’H˜U�ZÓ(ˆEØ—L‘L Ó,ˆEØ×Ñ˜UÖ#ñ	  ;ð
 Ðùó5
ùó AùÔ]s$   ÄI Ä&I Å/I,ÆI&ÆI,É&I,c	           
     óz  a aaaaaa€ \         P                  '       d   ^ RIoM\        R4      h\	        SR4      '       d   SP
                  oMo\	        SP                  P                  R4      '       d"   SP                  P                  P                  oM(\        S 4      R8”  d   \        P                  ! R4       Ro\        \        S VVVSRR7      oSP                  4        U	u. uF  p	SP                  P!                  V	4      NK   	  up	oSP#                  SP%                  RSP&                  4      .R	7      VVVV3R
 l4       p
SP(                  P*                  P-                  \        S 4      4      pV'       dM   SeI   SP/                  RSP1                  RSP&                  R7      R7      pV VV3R lpVP3                  WÍ4      pM'V'       d    VP5                  VP7                  4       4      pVe   VP9                  WxR7      pVP;                  V
4      pVe	   VV3R lpMVV3R lpVP;                  V4      # u up	i )aÄ  Create a tf.data.Dataset from the underlying Dataset. This is a single-process method - the multiprocess
equivalent is multiprocess_dataset_to_tf.

Args:
    dataset (`Dataset`): Dataset to wrap with tf.data.Dataset.
    cols_to_retain (`List[str]`): Dataset column(s) to load in the
        tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and
        that do not exist in the original dataset.
    collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
        lists of samples into a batch.
    collate_fn_args (`Dict`): A  `dict` of keyword arguments to be passed to the
        `collate_fn`. Can be empty.
    columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes.
    output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to
        `tf.TensorSpec` objects.
    shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
        validation/evaluation.
    batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that
        the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
    drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided,
        defaults to the same setting as shuffle.

Returns:
    `tf.data.Dataset`
Nr
   Úrandom_index_shuffleÚindex_shufflei€–˜ z½to_tf_dataset() can be memory-inefficient on versions of TensorFlow older than 2.9. If you are iterating over a dataset with a very large number of samples, consider upgrading to TF >= 2.9.F)r@   rA   rB   rC   rD   rE   )Úinput_signaturec                 ó¢   <€ SP                  SV .SR 7      p\        SP                  4       4       UUu/ uF  w  r#W1V,          bK  	  upp# u uppi ))ÚinpÚTout)Úpy_functionÚ	enumerateÚkeys)r?   ÚoutputrJ   rG   rD   Ú	getter_fnr   Útouts   &   €€€€r   Úfetch_functionÚ%dataset_to_tf.<locals>.fetch_function»   sT   ø€ à—‘ØØ�	Øð  ó 
ˆô
 .7Ð7J×7OÑ7OÓ7QÔ-RÔSÑ-R¡6 1�˜A•Y’Ñ-RÒSÐSùÓSs   ³A)Údtype)rH   c                 óÂ   <€ SP                  V R8H  4      '       d)   SP                  P                  RRSP                  R7      p S! W\	        S4      ^,
          R7      pW3# )é   )ÚshapeÚmaxvalr^   )ÚindexÚseedÚ	max_indexr0   ©é   l            )Ú
reduce_allÚrandomÚuniformÚint64r9   )Ústaterc   Úshuffled_indexr@   rP   r   s   && €€€r   Úscan_random_indexÚ(dataset_to_tf.<locals>.scan_random_indexÉ   sY   ø€ Ø�}‰}˜U b™[×)Ò)ð Ÿ	™	×)Ñ)°¸UÈ"Ï(É(Ð)ÓS�Ù1¸ÔUXÐY`ÓUaÐdeÕUeÔfˆNØÐ(Ð(r#   )Údrop_remainderc           
      óœ   <€ V P                  4        UUu/ uF)  w  rVSP                  VSV,          P                  4      bK+  	  upp# u uppi r%   ©r   Úensure_shapera   ©Ú
input_dictrG   ÚvalÚoutput_signaturer   s   &  €€r   Úensure_shapesÚ$dataset_to_tf.<locals>.ensure_shapesÜ   sD   ø€ Ø[e×[kÑ[kÔ[mÔnÑ[mÉxÈs�C˜Ÿ™¨Ð.>¸sÕ.C×.IÑ.IÓJÒJÑ[mÒnÐnùÓns   •/Ac           
      óª   <€ V P                  4        UUu/ uF0  w  rVSP                  VSV,          P                  R ,          4      bK2  	  upp# u uppi ):r`   NNrr   rt   s   &  €€r   rx   ry   á   sK   ø€ Ø_i×_oÑ_oÔ_qÔrÑ_qÑS[ÐSV�C˜Ÿ™¨Ð.>¸sÕ.C×.IÑ.IÈ"Õ.MÓNÒNÑ_qÒrÐrùÓrs   •6Arf   r0   )r   r   r   r   ÚhasattrrP   ri   ÚexperimentalrQ   r9   ÚwarningsÚwarnr   rN   r;   ÚdtypesÚas_dtypeÚfunctionÚ
TensorSpecrk   ÚdataÚDatasetr<   ÚfillÚcastÚscanÚshuffleÚcardinalityr   Úmap)r@   rA   rB   rC   rD   rw   rˆ   Ú
batch_sizerp   r^   r\   Ú
tf_datasetÚ	base_seedrn   rx   rZ   rP   r   r[   s   f&&&ff&&&      @@@@r   Údataset_to_tfrŽ   v   sÝ  þ€ ôH ××ÐÜäÐbÓcÐcô ˆrÐ)×*Ò*Ø!×6Ñ6ÑÜ	�—‘×'Ñ'¨×	9Ò	9Ø!Ÿy™y×5Ñ5×CÑCÑäˆw‹<˜*Ô$Ü�MŠMð*ôð
  $ÐäÜØØ%ØØ'Ø/Øô€Ið 4G×3MÑ3MÔ3OÓPÑ3O¨%ˆB�I‰I×Ñ˜uÖ%Ñ3OÑP€Dà‡[�[ "§-¡-°°b·h±hÓ"?Ð!@€[ÓA÷Tó BðTð —‘—‘×&Ñ&¤s¨7£|Ó4€JçÐ'Ò3Ø—G‘G˜D¨¯©°¸"¿(¹(¨Ó(C�GÓDˆ	÷	)ð  —_‘_ YÓB‰
ß	Ø×'Ñ'¨
×(>Ñ(>Ó(@ÓAˆ
àÒØ×%Ñ% jÐ%ÓPˆ
à—‘ Ó/€JàÒ÷	oö
	sð �>‰>˜-Ó(Ð(ùòW Qs   Ã$$H8c                   ó>   a € ] tR t^çt o R tR tR tR tR tRt	V t
R# )ÚSharedMemoryContextc                ó"   € . V n         . V n        R # r%   ©Úcreated_shmsÚopened_shms©Úselfs   &r   Ú__init__ÚSharedMemoryContext.__init__ê   s   € ØˆÔØˆÖr#   c                ó²   € \        \        V4      WR 7      pV'       d   V P                  P                  V4       V# V P                  P                  V4       V# ))ÚsizeÚnameÚcreate)r   Úintr“   r>   r”   )r–   r›   rš   rœ   Úshms   &&&& r   Úget_shmÚSharedMemoryContext.get_shmî   sK   € Ü¤ D£	°ÔDˆßà×Ñ×$Ñ$ SÔ)ð ˆ
ð ×Ñ×#Ñ# CÔ(Øˆ
r#   c                óà   € V P                  V\        P                  ! V4      \        P                  ! V4      P                  ,          VR 7      p\        P
                  ! W#VP                  R7      # ))r›   rš   rœ   )r^   Úbuffer)rŸ   r   Úprodr^   Úitemsizer   Úbuf)r–   r›   ra   r^   rœ   rž   s   &&&&& r   Ú	get_arrayÚSharedMemoryContext.get_arrayø   sG   € Ø�l‰l ¬2¯7ª7°5«>¼B¿HºHÀU»O×<TÑ<TÕ+TÐ]cˆlÓdˆÜ�zŠz˜%°S·W±WÔ=Ð=r#   c                ó   € V # r%   © r•   s   &r   Ú	__enter__ÚSharedMemoryContext.__enter__ü   ó   € Øˆr#   c                ó²   € V P                    F#  pVP                  4        VP                  4        K%  	  V P                   F  pVP                  4        K  	  R # r%   )r“   ÚcloseÚunlinkr”   )r–   Úexc_typeÚ	exc_valueÚ	tracebackrž   s   &&&& r   Ú__exit__ÚSharedMemoryContext.__exit__ÿ   s?   € Ø×$Ô$ˆCØ�I‰IŒKØ�J‰JŽLñ %ð ×#Ô#ˆCØ�I‰IŽKó $r#   r’   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r—   rŸ   r¦   rª   r³   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r�   r�   ç   s#   ø‡ € òòò>ò÷ð r#   r�   c                   óR   a € ] tR tRt o R tR tR t]R 4       t]R 4       t	Rt
V tR# )	ÚNumpyMultiprocessingGeneratori  c                ó®  € Wn         W n        W0n        W@n        VP	                  4        UUu. uF  w  r¼V\
        P                  J g   K  VNK  	  uppV n        VP	                  4        UUu/ uF.  w  r¼Y»V P                  9  d   TM\
        P                  ! R 4      bK0  	  uppV n	        W`n
        Wpn        W€n        W�n        W n        VP	                  4        UUu/ uF[  w  r½Y»V P                  9  d    \        VP                   P"                  4      M%\        VP                   P"                  4      ^,           bK]  	  uppV n        R# u uppi u uppi u uppi )ÚU1N)r@   rA   rB   rC   r   r   Ústr_Ústring_columnsr^   rD   rw   rˆ   r‹   rp   Únum_workersr�   ra   ÚrankÚcolumns_to_ranks)r–   r@   rA   rB   rC   rD   rw   rˆ   r‹   rp   rÃ   rL   r^   Úspecs   &&&&&&&&&&&   r   r—   Ú&NumpyMultiprocessingGenerator.__init__  s(  € ð ŒØ,ÔØ$ŒØ.ÔØ5H×5NÑ5NÔ5PÔeÑ5P¡z sÐTYÔ]_×]dÑ]dÐTdŸs˜sÑ5PÒeˆÔð 2×7Ñ7Ô9ô$
á9‘
�ð  T×%8Ñ%8Ô8‘¼b¿hºhÀt»nÒLÙ9ò$
ˆÔ ð !1ÔØŒØ$ŒØ,ÔØ&Ôð .×3Ñ3Ô5ô!
á5‘	�ð °D×4GÑ4GÔ)G”�T—Z‘Z—_‘_Ô%ÌSÐQU×Q[ÑQ[×Q`ÑQ`ÓMaÐdeÕMeÒeÙ5ò!
ˆÖùó fùó$
ùó!
s   ¬EÁ	EÁ+4EÃA!Ec              #  óv	  "  € \        V P                  \        \        \	        V P
                  4      V P                  ,          4      4      4      pV P                  V P
                  V P                  V P                  WP                  4      w  r#p\        R 4      p. p. p. p\        V4       U	u. uF  q•P                  4       NK  	  p
p	\        V4       U	u. uF  q•P                  4       NK  	  pp	RV P
                  RV P                  RV P                  RV P                  RV P                   RV P"                  RV P$                  /p\'        4       ;_uu_ 4       p\        V4       EF  p\)        \+        4       4      pRV R	V 2R
,          pVP-                  V4       V P"                  P/                  4        UUu/ uF1  w  ppVVP1                  V R	V R2V3\2        P4                  RR7      bK3  	  pppVP-                  V4       W.,          pWä8X  d   Ve   TpMRpRVRVRVRW®,          RW¾,          /VCpVP7                  V P8                  VRR7      pVP;                  4        VP-                  V4       EK  	  RpV'       EgÔ   \        V4       EFÀ  pW®,          P=                  ^<R7      '       g   \?        R4      hW®,          PA                  4        W~,          p\B        ;QJ d*    R VPE                  4        4       F  '       g   K   RM	  RM! R VPE                  4        4       4      '       d   Rp Kº  \'        4       ;_uu_ 4       pVP/                  4        UUu/ uF8  w  ppVVP1                  Wn,           R	V 2VV P                   V,          RR7      bK:  	  pppVP/                  4        UUu/ uF  w  ppV\2        PF                  ! V4      bK  	  pppV P$                   FH  pVV,          PI                  RVV,          PJ                  R,           24      PM                  R4      VV&   KJ  	  RRR4       Xx € W¾,          PO                  4        EKÃ  	  EKÜ  V F  pVPQ                  4        K  	  RRR4       R# u up	i u up	i u uppi u uppi u uppi   + '       g   i     Lq; i  + '       g   i     R# ; i5i)Úspawnr@   rA   rB   rC   rD   rÅ   rÂ   Údw_Ú_:Né
   NÚ_shapeT©ra   r^   rœ   NÚworker_namer?   Úextra_batchÚarray_ready_eventÚarray_loaded_event)ÚtargetÚkwargsÚdaemonF)ÚtimeoutzData loading worker timed out!c              3   óT   "  € T F  p\         P                  ! V^ 8  4      x € K   	  R# 5i)r	   N)r   Úany)Ú.0ra   s   & r   Ú	<genexpr>Ú9NumpyMultiprocessingGenerator.__iter__.<locals>.<genexpr>e  s"   é € ÐPÑ:O°œ2Ÿ6š6 %¨!¡)×,Ð,Ó:Oùs   ‚&(ÚUr0   ))ÚminrÃ   r�   r   r9   r@   r‹   Údistribute_batchesrp   rˆ   r   r<   ÚEventrA   rB   rC   rD   rÅ   rÂ   r�   Ústrr   r>   r   r¦   r   rk   ÚProcessÚworker_loopÚstartÚwaitÚTimeoutErrorÚclearrØ   r;   ÚcopyÚviewra   ÚsqueezeÚsetÚjoin)r–   rÃ   Úper_worker_batchesÚfinal_batchÚfinal_batch_workerÚctxÚnamesÚshape_arraysÚworkersrË   Úarray_ready_eventsÚarray_loaded_eventsÚ	base_argsÚshm_ctxrJ   Úworker_random_idrÏ   rL   rÄ   Úworker_shape_arraysÚworker_indicesÚfinal_batch_argÚworker_kwargsÚworkerÚend_signal_receivedÚarray_shapesÚbatch_shm_ctxra   ÚarraysÚarrÚ
string_cols   &                              r   Ú__iter__Ú&NumpyMultiprocessingGenerator.__iter__*  sS  é € ä˜$×*Ñ*¬C´´S¸¿¹Ó5FÈÏÉÕ5XÓ0YÓ,ZÓ[ˆà>B×>UÑ>UØ�L‰L˜$Ÿ/™/¨4×+>Ñ+>ÀÏ\É\ó?
Ñ;ÐÐ);ô ˜'Ó"ˆØˆØˆØˆÜ38¸Ô3EÓFÑ3E¨aŸi™ižkÑ3EÐÐFÜ49¸+Ô4FÓGÑ4F¨qŸy™yž{Ñ4FÐÐGð �t—|‘|Ø˜d×1Ñ1Ø˜$Ÿ/™/Ø˜t×3Ñ3Ø! 4×#;Ñ#;Ø × 5Ñ 5Ø˜d×1Ñ1ð
ˆ	ô !×"Ô" gÜ˜;×'�Ü#&¤u£w£<Ð Ø # A 3 aÐ(8Ð'9Ð:¸3Õ?�Ø—‘˜[Ô)ð &*×%:Ñ%:×%@Ñ%@Ô%Bô'á%B™	˜˜Tð ˜×*Ñ*¨k¨]¸!¸C¸5ÀÐ+GÐPTÈwÔ^`×^fÑ^fÐosÐ*ÓtÒtÙ%Bð $ñ 'ð ×#Ñ#Ð$7Ô8à!3Õ!6�ØÔ*¨{Ò/FØ&1‘Oà&*�Oà! ;Ø˜~Ø! ?Ø'Ð);Õ)>Ø(Ð*=Õ*@ð!ð  ð!�ð Ÿ™¨D×,<Ñ,<À]Ð[_˜Ó`�Ø—‘”Ø—‘˜v×&ñ5 (ð8 #(Ðß)Ð)Ü˜{×+�AØ-Õ0×5Ñ5¸bÐ5×AÒAÜ*Ð+KÓLÐLØ&Õ)×/Ñ/Ô1Ø#/¥?�Lß“sÑP¸,×:MÑ:MÔ:OÓP—s—s’sÑP¸,×:MÑ:MÔ:OÓP×PÒPð /3Ð+Úô -×.Ô.°-ð /;×.@Ñ.@Ô.Bô"ñ /C¡
  Uð   ×!8Ñ!8Ø#(¥8 *¨A¨c¨UÐ 3Ø&+Ø&*×&>Ñ&>¸sÕ&CØ',ð	 "9ó "ò ñ /Cð ñ "ð EKÇLÁLÄNÔ!SÁN¹¸¸S #¤r§w¢w¨s£|Ò"3ÁN˜Ñ!Sà*.×*=Ô*=˜Jà & zÕ 2× 7Ñ 7¸!¸FÀ:Õ<N×<TÑ<TÐUWÕ<XÐ;YÐ8ZÓ [× cÑ cÐdfÓ gð # :Ó.ñ +>÷ /ð& !’LØ'Õ*×.Ñ.×0ôK ,óP "�Ø—‘–ñ "÷O #Ñ"ùò GùÚGùó"'ùóX"ùó "T÷ /×.ú÷a #×"Ð"üs–   ‚B*R9Â,Q6ÃR9ÃQ;Ã+A*R9ÅA R%Æ57R Ç,BR%Ê B R%Ì(R%Ì,R%ÍRÍ>RÎRÎ+#RÏARÐ)AR%Ñ,R9Ò R%ÒRÒR"ÒR%Ò%R6	Ò0	R9c                ó   € V # r%   r©   r•   s   &r   Ú__call__Ú&NumpyMultiprocessingGenerator.__call__‹  r¬   r#   c                ó~  a aaaaaa	a
aa€ R \         P                  R&   \        P                  '       d   ^ RIpM\        R4      hVP                  P                  . R4       VV
VVVVV VVV	3
R lp\        4       ;_uu_ 4       pVP                  4        UUu/ uF0  w  ppWþP                  S	 RV R2V3\        P                  RR	7      bK2  	  uppoV F  pV! V4       K  	  Ve	   V! V4       SP                  4        F  w  ppRVR
&   K  	  S
P                  4        RRR4       R# u uppi   + '       g   i     R# ; i)Ú3ÚTF_CPP_MIN_LOG_LEVELNr
   ÚGPUc           
      ó.  <
€ \        V SSS	S
SR R7      p/ p\        4       ;_uu_ 4       pSP                  4        FŒ  w  rEW,          pVS9   d2   VP                  R4      P	                  VP
                  R,           4      pVP
                  SV,          R&   VP                  S RV 2VP
                  VR R7      W$&   WbV,          R&   KŽ  	  SP                  4        SP                  4        SP                  4        RRR4       R#   + '       g   i     R# ; i)T)r?   r@   rA   rB   rC   rD   rE   rÀ   ºNNNrË   rÎ   N)r0   )
rN   r�   r   rè   Úreshapera   r¦   rê   rä   ræ   )r?   r   Ú
out_arraysrÿ   rL   rM   r   rÒ   rÑ   rB   rC   rA   rD   r@   rñ   rÂ   rÏ   s   &      €€€€€€€€€€r   Úsend_batch_to_parentÚGNumpyMultiprocessingGenerator.worker_loop.<locals>.send_batch_to_parent¦  s  ø€ Ü ØØØ-Ø%Ø /Ø$7Ø ôˆEð ˆJÜ$×&Ô&¨-ð (;×'@Ñ'@Ö'B‘O�Cà!�J�EØ˜nÔ,ð !&§
¡
¨4Ó 0× 8Ñ 8¸¿¹ÀuÕ9LÓ M˜Ø+0¯;©;�L Õ% aÑ(Ø&3×&=Ñ&=Ø&˜- q¨¨Ð.°e·k±kÈÐ\`ð '>ó '�J‘Oð */˜s•O AÓ&ñ (Cð "×%Ñ%Ô'Ø"×'Ñ'Ô)Ø"×(Ñ(Ô*÷% '×&×&Ò&ús   ¨CDÄD	rË   rÍ   FrÎ   r  r0   )ÚosÚenvironr   r   r   r   Úset_visible_devicesr�   r   r¦   r   rk   rê   )r@   rA   rB   rC   rD   rÅ   rÂ   r?   rÐ   rÏ   rÑ   rÒ   r   r  rö   rL   rÄ   r   r   rñ   s   fffff&f&&fff       @r   râ   Ú)NumpyMultiprocessingGenerator.worker_loopŽ  s  ÿù€ ð .1Œ�
‰
Ð)Ñ*ä××ÐÜ#äÐfÓgÐgà
�	‰	×%Ñ% b¨%Ô0÷	+ö 	+ôB !×"Ô" gð "2×!7Ñ!7Ô!9ôá!9‘I�C˜ð ×&Ñ&¨+¨°a¸°u¸FÐ'CÈDÈ7ÔZ\×ZbÑZbÐkpÐ&ÓqÒqÙ!9òˆLó
 !�Ù$ UÖ+ñ !àÒ&Ù$ [Ô1à*×0Ñ0Ö2‘
��UØ��a“ñ 3à×!Ñ!Ô#÷ #Ñ"ùó÷ #×"Ð"ús   ÂD+Â6D%
ÃAD+Ä%D+Ä+D<	c                óP  € \         P                  ! \        V 4      4      pV'       d    \         P                  P	                  V4       \        V4      pWfV,          ,
          p\         P
                  ! WW.4      w  rXV'       g   \        V4      ^ 8X  d   RpVP                  RV4      p\        V4      p	W™V,          ,
          p
\         P
                  ! WZ.4      w  r[VP                  RW14      p\         P
                  ! WUP                  ^,          ^R7      pV Uu. uF  p\         P                  ! V^4      NK  	  pp\        \        V4      4       F;  p\         P                  ! WÎ,          W¾,          P                  ^R4      .^ R7      WÎ&   K=  	  Ve   \        V4      pMRpWÈV3# u upi )r	   N)Úaxisr0   )r   Úaranger9   ri   rˆ   Úsplitr  ra   ré   r<   Úconcatenate)r@   r‹   rp   rÃ   rˆ   r?   Únum_samplesÚincomplete_batch_cutoffÚlast_incomplete_batchÚnum_batchesÚfinal_batches_cutoffÚfinal_batchesÚper_worker_indicesrù   rJ   Úincomplete_batch_worker_idxs   &&&&&           r   rÞ   Ú0NumpyMultiprocessingGenerator.distribute_batchesÖ  sb  € ä—)’)œC ›LÓ)ˆßÜ�I‰I×Ñ˜gÔ&Ü˜'“lˆð #.¸zÕ1IÕ"JÐÜ)+¯ª°'Ð;TÓ)UÑ&ˆßœSÐ!6Ó7¸1Ô<Ø$(Ð!à—/‘/ " jÓ1ˆÜ˜'“lˆØ*¸KÕ.GÕHÐÜ!#§¢¨'Ð3IÓ!JÑˆØ—/‘/ " kÓ>ˆäŸXšX g¯}©}¸QÕ/?ÀaÔHÐÙRdÓeÑRdÀœbŸjšj¨¸Ö;ÑRdÐÐeä”s˜=Ó)Ö*ˆAä$&§N¢NÐ4FÕ4IÈ=ÕK[×KcÑKcÐdeÐgiÓKjÐ3kÐrsÔ$tÐÓ!ñ +ð !Ò,Ü*-¨mÓ*<Ñ'à*.Ð'Ø!Ð:UÐUÐUùò fs   Ä F#)r‹   rB   rC   rA   rD   rÅ   r@   rp   rÃ   rw   rˆ   rÂ   N)rµ   r¶   r·   r¸   r—   r  r  Ústaticmethodrâ   rÞ   r¹   rº   r»   s   @r   r¾   r¾     sD   ø‡ € ò 
òD_òBð ñE$ó ðE$ðN ñVó öVr#   r¾   c
                ó¼  € \         P                  '       d   ^ RIp
M\        R4      h\	        V VVVVVVVVV	R7
      pV
P
                  P                  P                  WµR7      pV'       d   \        \        V 4      V,          4      pM$\        \        \        V 4      V,          4      4      pVP                  V
P
                  P                  P                  V4      4      # )a  Create a tf.data.Dataset from the underlying Dataset. This is a multi-process method - the single-process
equivalent is dataset_to_tf.

Args:
    dataset (`Dataset`): Dataset to wrap with tf.data.Dataset.
    cols_to_retain (`List[str]`): Dataset column(s) to load in the
        tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and
        that do not exist in the original dataset.
    collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate
        lists of samples into a batch.
    collate_fn_args (`Dict`): A  `dict` of keyword arguments to be passed to the
        `collate_fn`. Can be empty.
    columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes.
    output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to
        `tf.TensorSpec` objects.
    shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for
        validation/evaluation.
    batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that
        the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`.
    drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided,
        defaults to the same setting as shuffle.
    num_workers (`int`): Number of workers to use for loading the dataset. Should be >= 1.

Returns:
    `tf.data.Dataset`
Nr
   )
r@   rA   rB   rC   rD   rw   rˆ   r‹   rp   rÃ   )rw   )r   r   r   r   r¾   rƒ   r„   Úfrom_generatorr�   r9   r   Úapplyr|   Úassert_cardinality)r@   rA   rB   rC   rD   rw   rˆ   r‹   rp   rÃ   r   Údata_generatorrŒ   Údataset_lengths   &&&&&&&&&&    r   Úmultiprocess_dataset_to_tfr+  ÷  s¹   € ôL ××ÐÜäÐbÓcÐcä2ØØ%ØØ'Ø/Ø)ØØØ%Øô€Nð —‘—‘×/Ñ/°Ð/Ób€JßÜœS ›\¨ZÕ7Ó8‰äœT¤# g£,°Õ";Ó<Ó=ˆØ×Ñ˜BŸG™G×0Ñ0×CÑCÀNÓSÓTÐTr#   )F)Ú__doc__r  r}   Ú	functoolsr   Úmathr   Úuuidr   r2   r   Úpyarrowr&   Úmultiprocessr   Úmultiprocess.shared_memoryr   r   Ú r   r   r"   r)   rN   rŽ   r�   r¾   r+  r©   r#   r   Ú<module>r4     sƒ   ðñ  ã 	Û Ý Ý Ý ã Û Ý $ðÝ7õ òò(òiô-ò`n)÷bñ ÷@mVñ mVô`=Uøðs ô Ø‚Lðús   ¬A" Á"	A.Á-A.