+
    QV-j2—  ã                   ó  € R t ^ RIt^ RIt^ RIt^ RIHt ^ RIHtHtH	t	 ^ RI
t
^ RIt^ RIt^ RIHt ^ RIHt ^ RIHt ^RIHt ^RIHtHtHtHtHtHt ^RIH t H!t!H"t"H#t#H$t$H%t%H&t&H't'H(t(H)t)H*t* ^RI+H,t, ^R	I-H.t. ^R
I/H0t0H1t1H2t2H3t3 ^RI4H5t5 ^RI6H7t7H8t8H9t9H:t:H;t;H<t<H=t=H>t> ^RI?H@t@HAtAHBtB ]5P†                  ! ]D4      tE]FtGR R ltHR R ltIR R ltJ ! R R]K4      tL ! R R4      tM ! R R]M4      tN ! R R4      tO ! R R]O4      tPR# )z$To write records into Parquet files.N)ÚIterable)ÚAnyÚLiteralÚOptional)Ú	url_to_fs)Úconfig)ÚAudioÚFeaturesÚImageÚPdfÚValueÚVideo)ÚFeatureTypeÚListÚ_ArrayXDExtensionTypeÚ_visitÚcast_to_python_objectsÚgenerate_from_arrow_typeÚget_nested_typeÚ%list_of_np_array_to_pyarrow_listarrayÚnumpy_to_pyarrow_listarrayÚrequire_storage_embedÚto_pyarrow_listarray)Úis_remote_filesystem)ÚDatasetInfo)Ú
array_castÚcast_array_to_featureÚembed_table_storageÚ
table_cast)Úlogging)Ú#find_mixed_struct_types_field_pathsÚ+get_json_field_path_from_pyarrow_json_errorÚ!get_json_field_paths_from_featureÚinsert_json_field_pathÚjson_encode_fieldÚ json_encode_fields_in_json_linesÚset_json_types_in_featureÚujson_dumps)ÚasdictÚconvert_file_size_to_intÚfirst_non_null_non_empty_valuec                ó\   € V ^8„  d   QhR\         \        ,          R\         \        ,          /# ©é   ÚfeaturesÚreturn©r   r	   Úint)Úformats   "Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/datasets/arrow_writer.pyÚ__annotate__r4   B   s$   € ÷ $8ñ $8¼ÄÕ8Jð $8ÌxÔX[Í}ñ $8ó    c                ó�   a€ V '       g   R# \         P                  oR V3R llp\        W4       S\         P                  J d   R# S# )a"  
Get the writer_batch_size that defines the maximum record batch size in the arrow files based on configuration values.
The default value is 100 for image/audio datasets and 10 for videos.
This allows to avoid overflows in arrow buffers.

Args:
    features (`datasets.Features` or `None`):
        Dataset Features from `datasets`.
Returns:
    writer_batch_size (`Optional[int]`):
        Writer batch size to pass to a dataset builder.
        If `None`, then it will use the `datasets` default, i.e. `datasets.config.DEFAULT_MAX_BATCH_SIZE`.
Nc                ó(   € V ^8„  d   QhR\         RR/# ©r-   Úfeaturer/   N©r   )r2   s   "r3   r4   Ú?get_arrow_writer_batch_size_from_features.<locals>.__annotate__U   s   € ÷ ]ñ ]¤ð ]°ñ ]r5   c                 óR  <€ \        V \        4      '       d/   \        P                  e   \	        S\        P                  4      oR # \        V \
        4      '       d/   \        P                  e   \	        S\        P                  4      oR # \        V \        4      '       d/   \        P                  e   \	        S\        P                  4      oR # \        V \        4      '       dD   V P                  R8X  d1   \        P                  e   \	        S\        P                  4      oR # R # R # R # ©NÚbinary)Ú
isinstancer
   r   Ú*ARROW_RECORD_BATCH_SIZE_FOR_IMAGE_DATASETSÚminr   Ú*ARROW_RECORD_BATCH_SIZE_FOR_AUDIO_DATASETSr   Ú*ARROW_RECORD_BATCH_SIZE_FOR_VIDEO_DATASETSr   ÚdtypeÚ+ARROW_RECORD_BATCH_SIZE_FOR_BINARY_DATASETS©r9   Ú
batch_sizes   &€r3   Úset_batch_sizeÚAget_arrow_writer_batch_size_from_features.<locals>.set_batch_sizeU   sÈ   ø€ ä�gœu×%Ò%¬&×*[Ñ*[Ò*gÜ˜Z¬×)ZÑ)ZÓ[ŠJÜ˜¤×'Ò'¬F×,]Ñ,]Ò,iÜ˜Z¬×)ZÑ)ZÓ[ŠJÜ˜¤×'Ò'¬F×,]Ñ,]Ò,iÜ˜Z¬×)ZÑ)ZÓ[ŠJä�w¤×&Ò&Ø—‘ Ô)Ü×BÑBÒNä˜Z¬×)[Ñ)[Ó\ŠJñ Oñ *ñ 'r5   ©ÚnpÚinfr   ©r.   rH   rG   s   & @r3   Ú)get_arrow_writer_batch_size_from_featuresrN   B   s@   ø€ ÷ Ùä—‘€J÷]ð ]ô ˆ8Ô$à¤§¡Ó'ˆ4Ð7¨ZÐ7r5   c                ó\   € V ^8„  d   QhR\         \        ,          R\         \        ,          /# r,   r0   )r2   s   "r3   r4   r4   i   s$   € ÷ %8ñ %8´(¼8Õ2Dð %8ÌÔRUÍñ %8r5   c                ó�   a€ V '       g   R# \         P                  oR V3R llp\        W4       S\         P                  J d   R# S# )ap  
Get the writer_batch_size that defines the maximum row group size in the parquet files based on configuration values.
By default these are not set, but it can be helpful to hard set those values in some cases.
This allows to optimize random access to parquet file, since accessing 1 row requires
to read its entire row group.

Args:
    features (`datasets.Features` or `None`):
        Dataset Features from `datasets`.
Returns:
    writer_batch_size (`Optional[int]`):
        Writer batch size to pass to a parquet writer.
        If `None`, then it will use the `datasets` default, i.e. aiming for row groups of 100MB.
Nc                ó(   € V ^8„  d   QhR\         RR/# r8   r:   )r2   s   "r3   r4   Ú9get_writer_batch_size_from_features.<locals>.__annotate__}   s   € ÷ \ñ \¤ð \°ñ \r5   c                 óR  <€ \        V \        4      '       d/   \        P                  e   \	        S\        P                  4      oR # \        V \
        4      '       d/   \        P                  e   \	        S\        P                  4      oR # \        V \        4      '       d/   \        P                  e   \	        S\        P                  4      oR # \        V \        4      '       dD   V P                  R8X  d1   \        P                  e   \	        S\        P                  4      oR # R # R # R # r=   )r?   r
   r   Ú)PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETSrA   r   Ú)PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETSr   Ú)PARQUET_ROW_GROUP_SIZE_FOR_VIDEO_DATASETSr   rD   Ú*PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETSrF   s   &€r3   rH   Ú;get_writer_batch_size_from_features.<locals>.set_batch_size}   sÈ   ø€ ä�gœu×%Ò%¬&×*ZÑ*ZÒ*fÜ˜Z¬×)YÑ)YÓZŠJÜ˜¤×'Ò'¬F×,\Ñ,\Ò,hÜ˜Z¬×)YÑ)YÓZŠJÜ˜¤×'Ò'¬F×,\Ñ,\Ò,hÜ˜Z¬×)YÑ)YÓZŠJä�w¤×&Ò&Ø—‘ Ô)Ü×AÑAÒMä˜Z¬×)ZÑ)ZÓ[ŠJñ Nñ *ñ 'r5   rJ   rM   s   & @r3   Ú#get_writer_batch_size_from_featuresrY   i   s@   ø€ ÷ Ùä—‘€J÷\ð \ô ˆ8Ô$à¤§¡Ó'ˆ4Ð7¨ZÐ7r5   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r-   Únum_rowsÚ	num_bytesr/   )r1   )r2   s   "r3   r4   r4   ‘   s&   € ÷ wñ w´3ð wÄ3ð wÌ3ñ wr5   c                óv   € V^ 8”  d2   \        ^V \        \        P                  4      ,          V,          4      # ^# )a  
Get the writer_batch_size that defines the maximum row group size in the parquet files.
The default in `datasets` is aiming for row groups of maximum 100MB uncompressed.
This allows to optimize random access to parquet file, since accessing 1 row requires
to read its entire row group.

This can be improved to get optimized size for querying/iterating
but at least it matches the dataset viewer expectations on HF.

Args:
    num_rows (`int`):
        Number of rows in the dataset.
    num_bytes (`int`):
        Number of bytes in the dataset.
        For dataset with external files to embed (image, audio, videos), this can also be an
        estimate from `dataset._estimate_nbytes()`.
Returns:
    writer_batch_size (`Optional[int]`):
        Writer batch size to pass to a parquet writer.
)Úmaxr)   r   ÚMAX_ROW_GROUP_SIZE)r[   r\   s   &&r3   Ú$get_writer_batch_size_from_data_sizer`   ‘   s8   € ð* clÐnoÔboŒ3ˆq�(Ô5´f×6OÑ6OÓPÕPÐT]Õ]Ó^ÐvÐuvÐvr5   c                   ó   € ] tR t^©tRtR# )ÚSchemaInferenceError© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__rc   r5   r3   rb   rb   ©   s   † Ûr5   rb   c                   ó”   a € ] tR t^­t o RtRV 3R lR lltV 3R lR lt]V 3R lR l4       tRV 3R	 lR
 llt	RV 3R lR llt
RtV tR# )ÚTypedSequencea  
This data container generalizes the typing when instantiating pyarrow arrays, tables or batches.

More specifically it adds several features:
- Support extension types like ``datasets.features.Array2DExtensionType``:
    By default pyarrow arrays don't return extension arrays. One has to call
    ``pa.ExtensionArray.from_storage(type, pa.array(data, type.storage_type))``
    in order to get an extension array.
- Support for ``try_type`` parameter that can be used instead of ``type``:
    When an array is transformed, we like to keep the same type as before if possible.
    For example when calling :func:`datasets.Dataset.map`, we don't want to change the type
    of each column by default.
- Better error message when a pyarrow array overflows.

Example::

    from datasets.features import Array2D, Array2DExtensionType, Value
    from datasets.arrow_writer import TypedSequence
    import pyarrow as pa

    arr = pa.array(TypedSequence([1, 2, 3], type=Value("int32")))
    assert arr.type == pa.int32()

    arr = pa.array(TypedSequence([1, 2, 3], try_type=Value("int32")))
    assert arr.type == pa.int32()

    arr = pa.array(TypedSequence(["foo", "bar"], try_type=Value("int32")))
    assert arr.type == pa.string()

    arr = pa.array(TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64")))
    assert arr.type == Array2DExtensionType((1, 3), "int64")

    table = pa.Table.from_pydict({
        "image": TypedSequence([[[1, 2, 3]]], type=Array2D((1, 3), "int64"))
    })
    assert table["image"].type == Array2DExtensionType((1, 3), "int64")

Nc                ó†   <€ V ^8„  d   QhRS[ RS[S[,          RS[S[,          RS[S[,          RS[S[R,          ,          /# )r-   ÚdataÚtypeÚtry_typeÚoptimized_int_typeÚon_mixed_typesÚuse_json)r   r   r   r   )r2   Ú__classdict__s   "€r3   r4   ÚTypedSequence.__annotate__Õ   sV   ø€ ÷ #ñ #áð#ñ ‘{Õ#ð#ñ ™;Õ'ð	#ñ
 %¡[Õ1ð#ñ !¡¨Õ!4Õ5ñ#r5   c                óÞ   € Ve   Ve   \        R4      hWn        W n        W0n        W@n        WPn        V P                  R JV n        VR J;'       d    VR J ;'       d    VR J V n        R V n        R # )Nz)You cannot specify both type and try_type)	Ú
ValueErrorrl   rm   rn   ro   rp   Útrying_typeÚtrying_int_optimizationÚ_inferred_type)Úselfrl   rm   rn   ro   rp   s   &&&&&&r3   Ú__init__ÚTypedSequence.__init__Õ   sx   € ð Ò Ò 4ÜÐHÓIÐIàŒ	ØŒ	Ø ŒØ"4ÔØ,ÔàŸ=™=°Ð4ˆÔØ'9ÀÐ'E×'kÐ'kÈ$ÐRVÈ,×'kÐ'kÐ[cÐgkÐ[kˆÔ$à"ˆÖr5   c                ó    <€ V ^8„  d   QhRS[ /# )r-   r/   r:   )r2   rr   s   "€r3   r4   rs   ì   s   ø€ ÷ #ñ #¡;ñ #r5   c                ób   € V P                   f   \        P                  ! V 4       V P                   # )a[  Return the inferred feature type.
This is done by converting the sequence to an Arrow array, and getting the corresponding
feature type.

Since building the Arrow array can be expensive, the value of the inferred type is cached
as soon as pa.array is called on the typed sequence.

Returns:
    FeatureType: inferred feature type of the sequence.
)rx   ÚpaÚarray©ry   s   &r3   Úget_inferred_typeÚTypedSequence.get_inferred_typeì   s'   € ð ×ÑÒ&Ü�HŠH�TŒNØ×"Ñ"Ð"r5   c                óL   <€ V ^8„  d   QhRS[ RS[S[ S[S[,          3,          /# )r-   rl   r/   )r   Útupler   r   )r2   rr   s   "€r3   r4   rs   ü   s+   ø€ ÷ $ñ $©Hð $¹¹xÉÑR]ÕI^Ð?^Õ9_ñ $r5   c           
     ó  € \         P                  '       Ed   R\        P                  9   Ed   ^ RIp\        V 4      w  r#\        W1P                  P                  4      '       d9   V  Uu. uF!  qDe   \        4       P                  V4      MRNK#  	  up\        4       3# \        V\        4      '       d�   \        V^ ,          VP                  P                  4      '       dT   V  UUu. uF1  qDe*   V Uu. uF  p\        4       P                  V4      NK  	  upMRNK3  	  upp\        \        4       4      3# \         P                  '       Ed   R\        P                  9   Ed   ^ RIp\        V 4      w  r#\        W6P                  P                  4      '       d9   V  Uu. uF!  qDe   \        4       P                  V4      MRNK#  	  up\        4       3# \        V\        4      '       d�   \        V^ ,          VP                  P                  4      '       dT   V  UUu. uF1  qDe*   V Uu. uF  p\        4       P                  V4      NK  	  upMRNK3  	  upp\        \        4       4      3# V R3# u upi u upi u uppi u upi u upi u uppi )a¤  Implement type inference for custom objects like PIL.Image.Image -> Image type.

This function is only used for custom python objects that can't be directly passed to build
an Arrow array. In such cases is infers the feature type to use, and it encodes the data so
that they can be passed to an Arrow array.

Args:
    data (Iterable): array of data to infer the type, e.g. a list of PIL images.

Returns:
    Tuple[Iterable, Optional[FeatureType]]: a tuple with:
        - the (possibly encoded) array, if the inferred feature type requires encoding
        - the inferred feature type if the array is made of supported custom objects like
            PIL images, else None.
ÚPILNÚ
pdfplumber)r   ÚPIL_AVAILABLEÚsysÚmodulesÚ	PIL.Imager*   r?   r
   Úencode_exampleÚlistr   ÚPDFPLUMBER_AVAILABLEr‡   ÚpdfÚPDFr   )rl   r†   Únon_null_idxÚnon_null_valueÚvalueÚxr‡   s   &      r3   Ú_infer_custom_type_and_encodeÚ+TypedSequence._infer_custom_type_and_encodeû   s  € ô" ××Ñ E¬S¯[©[Õ$8Ûä+IÈ$Ó+OÑ(ˆLÜ˜.¯)©)¯/©/×:Ò:ÙbfÓgÑbfÐY^Ò9Jœ›×.Ñ.¨uÔ5ÐPTÒTÑbfÑgÔinÓipÐpÐpÜ˜.¬$×/Ò/´J¸~ÈaÕ?PÐRU×R[ÑR[×RaÑRa×4bÒ4bákoôÙkoÐbgÒBS¹Ó>¹°1”U“W×+Ñ+¨AÖ.¹Ò>ÐY]Ò]Ñkoòäœ›“=ð!ð !ô ×&×&Ñ&¨<¼3¿;¹;Õ+FÛä+IÈ$Ó+OÑ(ˆLÜ˜.¯.©.×*<Ñ*<×=Ò=Ù`dÓeÑ`dÐW\Ò7Hœ›×,Ñ,¨UÔ3ÈdÒRÑ`dÑeÔgjÓglÐlÐlÜ˜.¬$×/Ò/´J¸~ÈaÕ?PÐR\×R`ÑR`×RdÑRd×4eÒ4eáimôÙimÐ`eÒ@Q±eÓ<±e°”S“U×)Ñ)¨!Ö,±eÒ<ÐW[Ò[Ñimòäœ›“;ðð ð �TˆzÐùò hùò ?ùóùò fùò =ùós<   Á('I'Ã$I1Ã0"I,Ä	I1Æ'I7ÈJÈ!"I<É	JÉ,I1É<Jc                óD   <€ V ^8„  d   QhRS[ S[P                  ,          /# ©r-   rm   ©r   r~   ÚDataType)r2   rr   s   "€r3   r4   rs   "  s   ø€ ÷ ñ ¡H©R¯[©[Õ$9ñ r5   c                óz   € V P                  VR 7      pV P                  f   \        VP                  4      V n        V# ))rm   )Ú_arrow_arrayrx   r   rm   )ry   rm   Úouts   && r3   Ú__arrow_array__ÚTypedSequence.__arrow_array__"  s7   € Ø×Ñ TÐÓ*ˆØ×ÑÒ&Ü":¸3¿8¹8Ó"DˆDÔØˆ
r5   c                óD   <€ V ^8„  d   QhRS[ S[P                  ,          /# r˜   r™   )r2   rr   s   "€r3   r4   rs   (  s    ø€ ÷ cñ c¡©"¯+©+Õ!6ñ cr5   c                ó
  € Ve   \        R4      h?V P                  pV P                  f'   V P                  f   V P	                  V4      w  q n        V P
                  f,   V P                  '       d   V P                  MV P                  pMV P
                  pVe   \        V4      MRpV P                  e   \        V P                  4      MRpRp. p \        V\        4      '       d+   \        W#4      p\        P                  P                  W74      # \        V\        P                   4      '       d   \#        V4      pEMf\        V\$        4      '       dF   V'       d>   \        \'        V4      ^,          \        P                   4      '       d   \)        V4      pEMRpTp	V P*                  R8X  d   Vf   \-        V	RR7      pMVe   \/        V4      pV'       d%   V F  p
V	 U	u. uF  p	\1        Wš4      NK  	  up	p	K   	  \        P2                  ! \5        V	RR7      4      pV'       d�   \        P6                  P9                  V.R.R	7      p\:        P<                  ! VP>                  4      p\A        VR,          V4      p\C        V\;        RV/4      PD                  4      pV^ ,          pV PF                  '       Ed€   \        PH                  PK                  VP                  4      '       d   VPM                  V4      pV# \        PH                  PO                  VP                  4      '       Ed   \        PH                  PK                  VP                  PP                  4      '       d#   \S        V\        PT                  ! V4      4      pV# \        PH                  PO                  VP                  PP                  4      '       dx   \        PH                  PK                  VP                  PP                  PP                  4      '       d5   \S        V\        PT                  ! \        PT                  ! V4      4      4      pV# Ve-   \W        W�V P                  '       * V P                  '       * R
7      pV# u up	i   \X        \        PZ                  P\                  \        PZ                  P^                  \        PZ                  P`                  3 Ed>   pT P                  '       g,   \        T\        PZ                  P`                  4      '       d   h T P                  '       Edä    \        T\        P                   4      '       d   \#        T4      u Rp?# \        T\$        4      '       dS   T'       dK   \b        ;QJ d    R T 4       F  '       g   K   RM	  RM! R T 4       4      '       d   \)        T4      u Rp?# Rp\        P2                  ! \5        TRR7      4      u Rp?#   \        PZ                  P^                   Ed    pR\e        T4      9   d   \g        R\i        T4       RT R24      RhT PF                  '       dd   R\e        T4      9   dT   \        Pj                  ! TPm                  4       4      Pn                  p\p        Ps                  RT R24       Xu Rp?u Rp?# T'       dQ   R\e        T4      9   dA   \        P2                  ! \5        TRRR7      4      pTe   \W        Y�RRR
7      pTu Rp?u Rp?# h Rp?ii ; iR\e        T4      9   d   \g        R\i        T4       RT R24      RhT PF                  '       d_   R\e        T4      9   dO   \        Pj                  ! TPm                  4       4      Pn                  p\p        Ps                  RT R24       Xu Rp?# T'       Ed`   R\e        T4      9   g"   R\e        T4      9   g   R\e        T4      9   Ed/    \        P2                  ! \5        TRRR7      4      pEMð  \        P^                  \        P\                  3 EdÈ   pT P*                  R8X  Ed°   R\e        T4      9   g"   R\e        T4      9   g   R\e        T4      9   Ed   RPu                  T Uu. uF  p\w        RT/4      NK  	  Mu upi up4      Py                  4       pT U
u. uF  p
R.T
,           NK  	  Mu up
i pp
\{        TT4      pRp  \|        P~                  ! \€        P‚                  ! T4      \|        P„                  ! RR7      R7      pMs  \        P^                   d\   pR\e        T4      9   dB   R\e        T4      9   d2   \‡        \e        T4      4      p
\‰        Yj4       \{        TT4      p Rp?Kª   Rp?MRp?ii ; iTeV   \:        P<                  ! TP>                  4      p\A        YÆ4      p\C        Y¼PD                  4      pT^ ,          pTu Rp?u Rp?# h h Rp?ii ; iTe   \W        Y�RRR
7      pTu Rp?# h Rp?ii ; i)z=This function is called when calling pa.array(typed_sequence)NzMTypedSequence is supposed to be used with pa.array(typed_sequence, type=None)FTrq   )Ú
allow_root)Úonly_1d_for_numpyÚobj)Únames)Úallow_primitive_to_strÚallow_decimal_to_strc              3   óV   "  € T F  p\        V\        P                  4      x € K!  	  R # 5i©N)r?   rK   Úndarray)Ú.0r“   s   & r3   Ú	<genexpr>Ú-TypedSequence._arrow_array.<locals>.<genexpr>|  s$   é € Ð@qÑlpÐchÄÈEÔSU×S]ÑS]×A^ÐA^Ólpùs   ‚')Úoverflowz There was an overflow with type zE. Try to reduce writer_batch_size to have batches smaller than 2GB.
(Ú)znot in rangezFailed to cast a sequence to z. Falling back to int64.zCould not convert)r£   Úoptimize_list_castingz cannot mix struct and non-structz	Expected zCould not convert Ú
)Úuse_threads)Úread_optionszJSON parse error: Column(z) changed from)Eru   rl   rm   rn   r•   rx   rv   r   ro   r?   r   r   r~   ÚExtensionArrayÚfrom_storagerK   rª   r   r�   r*   r   rp   r    r"   r$   r   r   ÚTableÚfrom_arraysr	   Úfrom_arrow_schemaÚschemar&   r   Úarrow_schemarw   ÚtypesÚis_int64ÚcastÚis_listÚ
value_typer   Úlist_r   Ú	TypeErrorÚlibÚArrowTypeErrorÚArrowInvalidÚArrowNotImplementedErrorÚanyÚstrÚOverflowErrorÚtype_rD   Úto_pandas_dtypeÚnameÚloggerÚinfoÚjoinr'   Úencoder%   ÚpajÚ	read_jsonÚioÚBytesIOÚReadOptionsr!   r#   )ry   rm   rl   Úpa_typeÚoptimized_int_pa_typeÚtrying_cast_to_python_objectsÚjson_field_pathsÚstorager�   ÚexamplesÚjson_field_pathÚpa_tabler.   r9   ÚeÚoptimized_int_pa_type_strÚeeÚexampleÚoriginal_batchÚbatchÚeees   &&                   r3   rœ   ÚTypedSequence._arrow_array(  s  € ð ÒÜÐlÓmÐmØØ�y‰yˆà�9‰9Ò §¡Ò!6Ø(,×(JÑ(JÈ4Ó(PÑ%ˆDÔ%Ø×ÑÒ&Ø$(×$4×$4Ð$4�4—=’=¸$¿)¹)‰Dà×&Ñ&ˆDØ+/Ò+;”/ $Ô'Àˆà8<×8OÑ8OÒ8[ŒO˜D×3Ñ3Ô4Ðaeð 	ð ).Ð%ØÐðO	ä˜'Ô#8×9Ò9Ü.¨tÓ=�Ü×(Ñ(×5Ñ5°gÓGÐGô ˜$¤§
¡
×+Ò+Ü0°Ó6’Ü˜D¤$×'Ò'¯D´ZÔ@^Ð_cÓ@dÐefÕ@gÔik×isÑis×5tÒ5tÜ;¸DÓA’à04Ð-Ø�à×&Ñ&¨*Ô4¸ºÜ'JÈ8Ð`dÔ'eÑ$ØÒ%Ü'HÈÓ'NÐ$ç#Û+;˜ÙaiÓ#jÑaiÐU]Ô$5°hÖ$PÑaiÑ#jšñ ,<ô —h’hÔ5°hÐRVÔWÓX�ç#Ü!Ÿx™x×3Ñ3°S°EÀ%ÀÐ3ÓI�HÜ'×9Ò9¸(¿/¹/ÓJ�HÜ7¸À½ÐIYÓZ�GÜ)¨(´H¸eÀWÐ=MÓ4N×4[Ñ4[Ó\�HØ" 1�+�Cà×+×+Ñ+Ü—8‘8×$Ñ$ S§X¡X×.Ò.ØŸ(™(Ð#8Ó9�Cð ˆJô —X‘X×%Ñ% c§h¡h×/Ó/Ü—x‘x×(Ñ(¨¯©×)<Ñ)<×=Ò=Ü(¨¬b¯hªhÐ7LÓ.MÓN˜ð ˆJô Ÿ™×)Ñ)¨#¯(©(×*=Ñ*=×>Ò>Ä2Ç8Á8×CTÑCTÐUX×U]ÑU]×UhÑUh×UsÑUs×CtÒCtÜ(¨¬b¯hªh´r·x²xÐ@UÓ7VÓ.WÓX˜ð ˆJð Ò!ô ,Ø¸$×:JÑ:JÔ6JÐei×euÑeuÔauô�ð ˆJùò7 $køô: Ü�F‰F×!Ñ!Ü�F‰F×ÑÜ�F‰F×+Ñ+ð	
õ ]	ð ×#×#Ð#¬
°1´b·f±f×6UÑ6U×(VÒ(VØà××ÑðÜ! $¬¯
©
×3Ò3Ü9¸$Ó?Õ?Ü# D¬$×/Ò/·D¿S»SÑ@qÑlpÓ@q¿S¿SºSÑ@qÑlpÓ@q×=qÒ=qÜDÀTÓJÕJà8<Ð5Ü!ŸxšxÔ(>¸tÐW[Ô(\Ó]Õ]øÜ—v‘v×*Ñ*õ Ø!¤S¨£VÔ+Ü+Ø>¼uÀT»{¸mð  LRð  STð  RUð  UVð  Wóà#ð$ð ×5×5Ð5¸.ÌCÐPQËFÔ:RÜ46·H²HÐ=R×=bÑ=bÓ=dÓ4e×4jÑ4jÐ1ÜŸ™Ø;Ð<UÐ;VÐVnÐoôð  #Ÿ
š
ß6Ð;NÔRUÐVWÓRXÔ;XÜ ŸhšhÜ2°4È4ÐglÔmó˜ð  Ò+Ü"7Ø #À$Ð]aô#˜Cð  #Ÿ
š
àûð+úð, œs 1›vÔ%Ü#Ø6´u¸T³{°mð  DJð  KLð  JMð  MNð  Oóàðð ×-×-Ð-°.ÄCÈÃFÔ2JÜ,.¯HªHÐ5J×5ZÑ5ZÓ5\Ó,]×,bÑ,bÐ)Ü—‘Ð;Ð<UÐ;VÐVnÐoÔpØ•
ß.Ð.Ø#¤s¨1£vÔ-Ð1SÔWZÐ[\ÓW]Ô1]ÐalÔpsÐtuÓpvÕavð$ÜŸ(š(Ô#9¸$ÐRVÐnsÔ#tÓu’CøÜŸ™¬×):Ñ):Ð;õ "à×*Ñ*¨jÕ8Ø,´°B³Ô7Ø=ÄÀRÃÔHØ&¬#¨b«'Õ1ð *.¯©ÑaeÓ3fÑaeÐV]´KÀÈÐ@PÖ4QÒaeùÔ3fÓ)g×)nÑ)nÓ)p˜Ù]mÓ+nÑ]mÈ/¨U¨G°o×,EÐ,EÒ]mùÒ+nÐ(Ð+nÜ @ÀÐQaÓ b˜Ø#'˜Ø"ð*Ü+.¯=ª=Ü$&§J¢J¨uÓ$5ÄCÇOÂOÐ`eÔDfô," ð !&øÜ#%§?¡?ô *Ø#>Ä#ÀcÃ(Ô#JÐO_ÔcfÐgjÓckÔOkÜ6aÔbeÐfiÓbjÓ6k OÜ$:Ð;KÔ$]Ü,LÈ^Ð]mÓ,n¦Eä$)ûð*úð $Ò/Ü'/×'AÒ'AÀ(Ç/Á/Ó'R˜HÜ'@ÀÓ'\˜HÜ'1°(×<QÑ<QÓ'R˜HØ"*¨1¥+˜CØ#&ŸJšJà!àûðE"úðF Ò#Ü/°ÐRVÐmqÔr�CØ•
àûð{]	ús²  Ã?Q- ÄA
Q- ÅA:Q- Ç	Q- Ç Q(Ç4CQ- Ê7.Q- Ë&Q- Ë9/Q- Ì)8Q- Í"!Q- Î8Q- Î=AQ- Ð 5Q- Ð61Q- Ñ(Q- Ñ-AhÓAg=Ô*WÕ hÕWÕ$
WÕ/WÖWÖ
WÖ(hÖ."W×h× [5×6=[0Ø4A[0Ú[5Úg=ÚhÚ[0ÚA[0Û$[5Û%g=Û)hÛ/[0Û0[5Û5A g=Ü6Ag=ÞhÞg=Þ1g=ß"_3ß1g=ß3&g àAgá+bâgâ b4â3gãAdägäe=ä"Ae8å-gå8e=å=Agçg çg=çhçgçg ç g=ç6hç<g=ç=h)rx   rl   rp   ro   rn   rw   rv   rm   )NNNNr©   )rd   re   rf   rg   Ú__doc__rz   r�   Ústaticmethodr•   rž   rœ   rh   Ú__classdictcell__©rr   s   @r3   rj   rj   ­   sP   ø‡ € ñ%÷N#ò #÷.#ð #ð ÷$ó ð$÷Lò ÷c÷ cð cr5   rj   c                   óB   a a€ ] tR tRt oRV3R lV 3R llltRtVtV ;t# )ÚOptimizedTypedSequenceiÎ  c                ó–   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          RS[ S[,          RS[ S[,          RS[ S[R,          ,          /# )r-   rm   rn   Úcolro   rp   rq   )r   r   rÇ   r   )r2   rr   s   "€r3   r4   Ú#OptimizedTypedSequence.__annotate__Ï  s\   ø€ ÷ 
ñ 
ñ ‘{Õ#ð
ñ ™;Õ'ð	
ñ
 ‘c�]ð
ñ %¡[Õ1ð
ñ !¡¨Õ!4Õ5ñ
r5   c           
     ó¼   <€ R \        R4      R\        R4      R\        R4      R\        R4      /pVf   Vf   VP                  VR4      p\        SV `  WW5VR7       R# )Úattention_maskÚint8Úspecial_tokens_maskÚ	input_idsÚint32Útoken_type_idsN)rm   rn   ro   rp   )r   ÚgetÚsuperrz   )	ry   rl   rm   rn   rì   ro   rp   Úoptimized_int_type_by_colÚ	__class__s	   &&&&&&& €r3   rz   ÚOptimizedTypedSequence.__init__Ï  sn   ø€ ð œe F›mØ!¤5¨£=Øœ˜w›ØœeØóð	%
Ð!ð Š<˜HÒ,Ø!:×!>Ñ!>¸sÀDÓ!IÐÜ‰ÑØ hÐftð 	ö 	
r5   rc   )NNNNN)rd   re   rf   rg   rz   rh   rç   Ú__classcell__©rø   rr   s   @@r3   rê   rê   Î  s   ù‡ € ÷
÷ 
õ 
r5   rê   c                   ó@  a € ] tR tRt o RtR!V 3R lR lltR tR tR tR	 t	V 3R
 lR lt
V 3R lR lt]R 4       t]R"V 3R lR ll4       tR tR tR"V 3R lR lltR"V 3R lR lltR#V 3R lR lltR#V 3R lR lltR"V 3R lR lltR"V 3R lR lltR$R ltR tV tR# )%ÚArrowWriteriç  z,Shuffles and writes Examples to Arrow files.Nc                ó  <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[,          RS[ S[,          RS[ S[P
                  ,          RS[ S[,          RS[ S[,          RS[RS[R	S[ S[R
,          ,          RS[RS[RS[RS[ S[	,          /# )r-   r¹   r.   ÚpathÚstreamÚfingerprintÚwriter_batch_sizeÚdisable_nullableÚupdate_featuresrp   rq   Úwith_metadataÚunitÚembed_local_filesÚstorage_options)
r   r~   ÚSchemar	   rÇ   Ú
NativeFiler1   Úboolr   Údict)r2   rr   s   "€r3   r4   ÚArrowWriter.__annotate__ê  s¾   ø€ ÷ <ñ <á™Ÿ™Õ#ð<ñ ™8Õ$ð<ñ ‘s�mð	<ñ
 ™Ÿ™Õ'ð<ñ ™c•]ð<ñ $¡C�=ð<ñ ð<ñ ð<ñ !¡¨Õ!4Õ5ð<ñ ð<ñ ð<ñ  ð<ñ "¡$�ñ<r5   c                ó>  € Vf   Vf   \        R4      hVe   W n        R V n        M>Ve-   Wn        \        P                  ! V P                  4      V n        MR V n        R V n        Wpn        Vf…   \        V3/ T;'       g    / B w  rãWàn        \        V P                  4      '       g   TMV P                  P                  V4      V n
        V P                  P                  VR4      V n        RV n        MR V n        R V n
        W@n        RV n        WPn        Wpn        T;'       g.    \!        V P                  4      ;'       g    \"        P$                  V n        W€n        W�n        W n        W°n        WÀn        ^ V n        ^ V n        . V n        . V n        R V n        . V n        R # )Nz1At least one of path and stream must be provided.ÚwbTF)ru   Ú	_featuresÚ_schemar	   r¸   Ú_disable_nullabler   Ú_fsr   Úunstrip_protocolÚ_pathÚopenr   Ú_closable_streamr  r  rN   r   ÚDEFAULT_MAX_BATCH_SIZEr  r  rp   r  r  r  Ú_num_examplesÚ
_num_bytesÚcurrent_examplesÚcurrent_rowsÚ	pa_writerÚhkey_record)ry   r¹   r.   rÿ   r   r  r  r  r  rp   r  r  r  r  Úfss   &&&&&&&&&&&&&& r3   rz   ÚArrowWriter.__init__ê  sp  € ð  Š<˜FšNÜÐPÓQÐQØÒØ%ŒNØˆD�LØÒØ&,ŒLÜ%×7Ò7¸¿¹ÓEˆD�Nà!ˆDŒNØˆDŒLà!1ÔàŠ>Ü  ÑA¨/×*?Ð*?¸RÑA‰HˆBØ24ŒHÜ%9¸$¿(¹(×%CÒ%C™ÈÏÉ×IbÑIbÐcgÓIhˆDŒJØŸ(™(Ÿ-™-¨¨dÓ3ˆDŒKØ$(ˆDÕ!àˆDŒHØˆDŒJØ ŒKØ$)ˆDÔ!à&ÔØ 0Ôà÷ -ð -Ü8¸¿¹ÓH÷-ð -ä×,Ñ,ð 	Ôð
  /ÔØ,ÔØ*ÔØŒ	Ø!2ÔàˆÔØˆŒØBDˆÔØ,.ˆÔØ?CˆŒØˆÖr5   c                ó‚   € V P                   \        V P                  4      ,           \        V P                  4      ,           # )z/Return the number of writed and staged examples)r  Úlenr  r  r€   s   &r3   Ú__len__ÚArrowWriter.__len__(  s.   € à×!Ñ!¤C¨×(=Ñ(=Ó$>Õ>ÄÀT×EVÑEVÓAWÕWÐWr5   c                ó   € V # r©   rc   r€   s   &r3   Ú	__enter__ÚArrowWriter.__enter__,  s   € Øˆr5   c                ó&   € V P                  4        R # r©   )Úclose)ry   Úexc_typeÚexc_valÚexc_tbs   &&&&r3   Ú__exit__ÚArrowWriter.__exit__/  s   € Ø�
‰
Žr5   c                ó  € V P                   '       d    V P                   P                  4        V P                  '       d;   V P                  P
                  '       g   V P                  P                  4        R # R # R #   \         d     L[i ; ir©   )r  r)  Ú	Exceptionr  r   Úclosedr€   s   &r3   r)  ÚArrowWriter.close2  si   € à�>�>ˆ>ðØ—‘×$Ñ$Ô&ð × × Ð ¨¯©×);×);Ð);Ø�K‰K×ÑÖñ *<Ñ øô ô Ùðús   ”A< Á<B
Â	B
c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r-   Úinferred_schema©r~   r	  )r2   rr   s   "€r3   r4   r  <  s   ø€ ÷  ñ  ©R¯Y©Yñ  r5   c                óÜ  € V P                   pV P                  p\        P                  ! V4      pV P                  e’   V P                  '       d   V P                  P
                   Uu/ uF  qUP                  VbK  	  ppVP
                   F;  pVP                  pW†9   g   K  WvV,          8X  g   K&  V P                  V,          WH&   K=  	  TpTpMTpVP                  pV P                  '       d   \        P                   ! R V 4       4      pV P                  '       d9   VP                  V P                  \        VR7      V P                  4      4      pW#3# VP                  / 4      pW#3# u upi )Nc              3   ó|   "  € T F2  p\         P                  ! VP                  VP                  R R7      x € K4  	  R# 5i©F)ÚnullableN©r~   ÚfieldrË   rm   ©r«   r<  s   & r3   r¬   Ú,ArrowWriter._build_schema.<locals>.<genexpr>O  s+   é € ÐdÑ]cÐTYœrŸxšx¨¯
©
°E·J±JÈ×OÑOÓ]cùó   ‚:<)r.   )r¹   r  r	   r¸   r  rm   rË   rº   r  r~   r  Ú_build_metadatar   r  )	ry   r5  r¹   r.   Úinferred_featuresr<  ÚfieldsÚinferred_fieldrË   s	   &&       r3   Ú_build_schemaÚArrowWriter._build_schema<  s>  € Ø—‘ˆØ—>‘>ˆÜ$×6Ò6°ÓGÐØ�>‰>Ò%Ø×#×#Ð#Ø9=¿¹×9LÒ9LÓMÑ9L°Ÿ*™* eÒ+Ñ9L�ÐMØ&7×&<Ô&<�NØ)×.Ñ.�DØ–~Ø)°D­\Ö9Ø6:·n±nÀTÕ6JÐ-Ó3ñ	 '=ð
 -�Ø$3�øà(ˆHØ 1× >Ñ >ˆFà× × Ð Ü—Y’YÑdÑ]cÓdÓdˆFØ××ÐØ×)Ñ)¨$×*>Ñ*>¼{ÐT\Ô?]Ð_c×_oÑ_oÓ*pÓqˆFð ÐÐð ×)Ñ)¨"Ó-ˆFàÐÐùò' Ns   Á'E)c                ó4   <€ V ^8„  d   QhRS[ P                  /# r4  r6  )r2   rr   s   "€r3   r4   r  W  s   ø€ ÷ Oñ O©R¯Y©Yñ Or5   c                ó¢   € V P                  V4      w  V n        V n        \        P                  ! V P
                  V P                  4      V n        R # r©   )rD  r  r  r~   ÚRecordBatchStreamWriterr   r  )ry   r5  s   &&r3   Ú_build_writerÚArrowWriter._build_writerW  s7   € Ø'+×'9Ñ'9¸/Ó'JÑ$ˆŒ�d”nÜ×3Ò3°D·K±KÀÇÁÓNˆŽr5   c                ó   € V P                   e   V P                   M9V P                  e+   \        P                  ! V P                  P                  4      MR pV P
                  '       d"   Ve   \        P                  ! R V 4       4      pVe   V# . # )Nc              3   ó|   "  € T F2  p\         P                  ! VP                  VP                  R R7      x € K4  	  R# 5ir9  r;  r=  s   & r3   r¬   Ú%ArrowWriter.schema.<locals>.<genexpr>c  s+   é € ÐfÑ^eÐUZ¤§¢¨¯©°U·Z±ZÈ%× PÑ PÓ^eùr?  )r  r  r~   r¹   rm   r  )ry   r  s   & r3   r¹   ÚArrowWriter.schema[  sv   € ð �|‰|Ò'ð �LŠLà48·N±NÒ4N”"—)’)˜DŸN™N×/Ñ/Ô0ÐTXð 	ð
 ×!×!Ð! gÒ&9Ü—i’iÑfÑ^eÓfÓfˆGØ!Ò-ˆwÐ5°2Ð5r5   c                óR   <€ V ^8„  d   QhRS[ RS[S[,          RS[S[S[3,          /# )r-   rÍ   r  r/   )r   r   rÇ   r  )r2   rr   s   "€r3   r4   r  g  s3   ø€ ÷ 5ñ 5™kð 5¹Á½ð 5ÑQUÑVYÑ[^ÐV^ÕQ_ñ 5r5   c                ó¢   € R .p\        V 4      p/ pV Uu/ uF  qUW5,          bK  	  upVR&   Ve   WR&   R\        P                  ! V4      /# u upi )r.   rÍ   r  Úhuggingface)r(   ÚjsonÚdumps)rÍ   r  Ú	info_keysÚinfo_as_dictÚmetadataÚkeys   &&    r3   r@  ÚArrowWriter._build_metadataf  s^   € à�Lˆ	Ü˜d“|ˆØˆÙ>GÓH¹i°s Õ!2Ò2¹iÑHˆ�ÑØÒ"Ø&1�]Ñ#ØœtŸzšz¨(Ó3Ð4Ð4ùò Is   •Ac           	     ó&  a€ V P                   '       g   R# V P                  '       d—   \        V P                  P                  4      pV P                   ^ ,          ^ ,          P	                  4       pV P                  P                   Uu. uF  q3V9   g   K  VNK  	  ppV Uu. uF  q3V9  g   K  VNK  	  ppWE,           pM#\        V P                   ^ ,          ^ ,          4      p/ pV EFt  o\        ;QJ d)    V3R lV P                    4       F  '       d   K   RM	  RM! V3R lV P                    4       4      '       d�   V P                    Uu. uF  qˆ^ ,          S,          NK  	  p	pV	 U
Uu. uF;  p
\        V
\        P                  4      '       d   V
P                  MV
. F  pVNK  	  K=  	  p	p
p\        P                  ! V	4      VS&   Kä  V P                    Uu. uFv  p\        V^ ,          S,          \        P                  \        P                  34      '       d&   V^ ,          S,          P                  4       ^ ,          MV^ ,          S,          NKx  	  upVS&   EKw  	  V P                  VR7       . V n         R# u upi u upi u upi u upp
i u upi )ziWrite stored examples from the write-pool of examples. It makes a table out of the examples and write it.Nc              3   ó”   <"  € T F=  p\        V^ ,          S,          \        P                  \        P                  34      x € K?  	  R# 5i)é    N)r?   r~   ÚArrayÚChunkedArray)r«   Úrowrì   s   & €r3   r¬   Ú5ArrowWriter.write_examples_on_file.<locals>.<genexpr>‚  s2   øé € ÐiÑShÈC”:˜c !�f S�k¬B¯H©H´b·o±oÐ+F×GÐGÓShùs   ƒAAFT)Úbatch_examples)r  r¹   Úsetr¥   Úkeysr�   Úallr?   r~   r]  ÚchunksÚconcat_arraysr\  Ú	to_pylistÚ_write_batch)ry   Úschema_colsÚexamples_colsrì   Úcommon_colsÚ
extra_colsÚcolsr`  r^  Úarraysr   Úchunks   &  `        r3   Úwrite_examples_on_fileÚ"ArrowWriter.write_examples_on_filep  s"  ø€ à×$×$Ð$Ùà�;�;ˆ;Ü˜dŸk™k×/Ñ/Ó0ˆKØ ×1Ñ1°!Õ4°QÕ7×<Ñ<Ó>ˆMØ*.¯+©+×*;Ò*;ÓTÑ*; 3ÀmÑ?SŸ3˜3Ñ*;ˆKÐTÙ)6ÓQ© #À[Ñ:PŸ#˜#©ˆJÐQØÕ+‰Dä˜×-Ñ-¨aÕ0°Õ3Ó4ˆDØˆÜˆC÷ ‹sÔiÐSW×ShÒShÓi�s�sŠsÔiÐSW×ShÒShÓi×iÒiØ15×1FÒ1FÓGÑ1F¨#˜a�& Ÿ+˜+Ñ1F�ÐGñ "(ôá!'˜Ü2<¸UÄBÇOÁO×2TÒ2T %§,¢,Ð[`ÐZaÒ"a˜ó á"añ Ù!'ð ñ ô
 ')×&6Ò&6°vÓ&>�˜sÓ#ð  $×4Ò4ó'á4˜ô 3=¸SÀ½VÀC½[Ì2Ï8É8ÔUW×UdÑUdÐJe×2fÒ2f�C˜•F˜3•K×)Ñ)Ó+¨AÖ.ÐloÐpqÕlrÐsvÕlwÒwÙ4ñ'�˜sÔ#ñ ð" 	×Ñ¨ÐÔ8Ø "ˆÖùò1 UùÚQùò Hùóùò's,   ÂI9ÂI9Â I>Â-I>ÅJÅ*AJÇA<Jc                óž   € V P                   '       g   R# \        P                  ! V P                   4      pV P                  V4       . V n         R# )zwWrite stored rows from the write-pool of rows. It concatenates the single-row tables and it writes the resulting table.N)r  r~   Úconcat_tablesÚ_write_table)ry   Útables   & r3   Úwrite_rows_on_fileÚArrowWriter.write_rows_on_file’  s=   € à× × Ð ÙÜ× Ò  ×!2Ñ!2Ó3ˆØ×Ñ˜%Ô ØˆÖr5   c                óL   <€ V ^8„  d   QhRS[ S[S[3,          RS[S[,          /# )r-   rà   r  )r  rÇ   r   r   r1   )r2   rr   s   "€r3   r4   r  š  s+   ø€ ÷ *ñ *á‘c™3�h•ð*ñ $¡C�=ñ*r5   c                óÄ   € V P                   P                  VR34       Vf   V P                  pVe/   \        V P                   4      V8¼  d   V P	                  4        R# R# R# )zAdd a given (Example,Key) pair to the write-pool of examples which is written to file.

Args:
    example: the Example to add.
Ú N)r  Úappendr  r"  ro  )ry   rà   r  s   &&&r3   ÚwriteÚArrowWriter.writeš  s\   € ð 	×Ñ×$Ñ$ g¨r ]Ô3àÒ$Ø $× 6Ñ 6ÐØÒ(¬S°×1FÑ1FÓ-GÐK\Ô-\Ø×'Ñ'Ö)ñ .]Ñ(r5   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r-   r^  r  ©r~   r¶   r   r1   )r2   rr   s   "€r3   r4   r  ¬  s#   ø€ ÷ &ñ &™RŸX™Xð &¹(Á3½-ñ &r5   c                ó  € \        V4      ^8w  d   \        R\        V4       R24      hV P                  P                  V4       Vf   V P                  pVe/   \        V P                  4      V8¼  d   V P                  4        R# R# R# )zqAdd a given single-row Table to the write-pool of rows which is written to file.

Args:
    row: the row to add.
z>Only single-row pyarrow tables are allowed but got table with z rows.N)r"  ru   r  rz  r  ru  )ry   r^  r  s   &&&r3   Ú	write_rowÚArrowWriter.write_row¬  s�   € ô ˆs‹8�qŒ=ÜÐ]Ô^aÐbeÓ^fÐ]gÐgmÐnÓoÐoØ×Ñ× Ñ  Ô%ØÒ$Ø $× 6Ñ 6ÐØÒ(¬S°×1BÑ1BÓ-CÐGXÔ-XØ×#Ñ#Ö%ñ .YÑ(r5   c                ób   <€ V ^8„  d   QhRS[ S[S[3,          RS[S[,          RS[S[,          /# ©r-   r`  r  Útry_original_type©r  rÇ   r�   r   r1   r  )r2   rr   s   "€r3   r4   r  º  s>   ø€ ÷ tñ tá™S¡$˜Y�ðtñ $¡C�=ðtñ $¡D�>ñ	tr5   c                óL   € V P                  4        V P                  WVR7       R# )a7  Write a batch of Example to file.
Ignores the batch if it appears to be empty,
preventing a potential schema update of unknown types.

Args:
    batch_examples: the batch of examples to add.
    try_original_type: use `try_type` when instantiating OptimizedTypedSequence if `True`, otherwise `try_type = None`.
)r  r„  N)ro  rg  )ry   r`  r  r„  s   &&&&r3   Úwrite_batchÚArrowWriter.write_batchº  s$   € ð 	×#Ñ#Ô%Ø×Ñ˜.ÐarÐÖsr5   c                ób   <€ V ^8„  d   QhRS[ S[S[3,          RS[S[,          RS[S[,          /# rƒ  r…  )r2   rr   s   "€r3   r4   r  Ë  s9   ø€ ÷ )6ñ )6á™S¡$˜Y�ð)6ñ $¡C�=ð)6ñ $¡D�>ñ	)6r5   c           	     ó8  € V'       d3   \        \        \        VP                  4       4      4      4      ^ 8X  d   R# V P                  f   V P
                  '       d   RMV P                  pV P                  f   V P
                  '       d   V P                  MRp. p\        4       pV P                  '       d   \        V P                  P                  4      pVP                  4       p	V P                  P                   U
u. uF  qªV	9   g   K  V
NK  	  pp
V	 U
u. uF  qªV9  g   K  V
NK  	  pp
W¼,           pM\        V4      pV Fó  p
W,          pV'       d	   WJ,          MRp\        V\        P                  \        P                   34      '       d<   Ve   \#        Wï4      MTpVP%                  V4       \'        VP(                  4      Wz&   Kˆ  Ve   W¥9   d   V'       d	   WZ,          MRp\+        WïVW P,                  R7      pVP%                  \        P.                  ! V4      4       VP1                  4       Wz&   Kõ  	  V P                  f   VP2                  MV P                  p\        P4                  P7                  VVR7      pV P9                  VV4       R# u up
i u up
i )r[  N)rm   rn   rì   rp   )r¹   )r"  ÚnextÚiterÚvaluesr  r  r  r	   r¹   ra  r¥   rb  r�   r?   r~   r\  r]  r   rz  r   rm   rê   rp   r   r�   rº   r¶   r·   Úwrite_table)ry   r`  r  r„  r.   Útry_featuresrm  rA  rh  Ú
batch_colsrì   rj  rk  rl  Ú
col_valuesÚcol_typer   Úcol_try_typeÚtyped_sequencer¹   rÜ   s   &&&&                 r3   rg  ÚArrowWriter._write_batchË  s  € ÷ œc¤$¤t¨N×,AÑ,AÓ,CÓ'DÓ"EÓFÈ!ÔKÙØŸ>™>Ò1°d×6J×6JÐ6J‘4ÐPT×P^ÑP^ˆØ)-¯©Ò)?ÀD×DX×DXÐDX�t—~’~Ð^bˆØˆÜ$›JÐà�;�;ˆ;Ü˜dŸk™k×/Ñ/Ó0ˆKØ'×,Ñ,Ó.ˆJØ*.¯+©+×*;Ò*;ÓQÑ*; 3ÀjÑ?PŸ3˜3Ñ*;ˆKÐQÙ)3ÓN© #À+Ñ7MŸ#˜#©ˆJÐNØÕ+‰Dä˜Ó'ˆDÛˆCØ'Õ,ˆJß(0�x–}°dˆHÜ˜*¤r§x¡x´·±Ð&A×BÒBØGOÒG[Ô-¨jÔCÐak�Ø—‘˜eÔ$Ü)AÀ*Ç/Á/Ó)RÐ!Ó&ð $Ò/°CÔ4G×L]ð !Ö%àð ô
 "8Ø¸È#×^qÑ^qô"�ð —‘œbŸhšh ~Ó6Ô7Ø)7×)IÑ)IÓ)KÐ!Ó&ñ# ð$ 48·>±>Ò3IÐ"×/Ò/ÈtÏ{É{ˆÜ—8‘8×'Ñ'¨°vÐ'Ó>ˆØ×Ñ˜Ð#4Ö5ùò3 RùÚNs   Ã<JÄ	JÄJÄ"Jc                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# ©r-   rÜ   r  r~  )r2   rr   s   "€r3   r4   r  ö  s'   ø€ ÷ Iñ I¡B§H¡Hð IÁÉ#Åñ Ir5   c                óJ   € V P                  4        V P                  WR7       R# )z=Write a Table to file.

Args:
    example: the Table to add.
)r  N)ru  rs  ©ry   rÜ   r  s   &&&r3   rŽ  ÚArrowWriter.write_tableö  s!   € ð 	×ÑÔ!Ø×Ñ˜(ÐÖHr5   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# r—  r~  )r2   rr   s   "€r3   r4   r  ÿ  s'   ø€ ÷ @ñ @¡R§X¡Xð @Á(É3Å-ñ @r5   c                óÆ  € Vf   V P                   pV P                  f   V P                  VP                  R7       VP	                  4       p\        WP                  4      pV P                  '       d   \        VRRR7      pV ;P                  VP                  ,          un	        V ;P                  VP                  ,          un        V P                  P                  W4       R # )N)r5  TF)Úlocal_filesÚremote_files)r  r  rI  r¹   Úcombine_chunksr   r  r  r   r  Únbytesr  r[   rŽ  r™  s   &&&r3   rs  ÚArrowWriter._write_tableÿ  s¥   € ØÒ$Ø $× 6Ñ 6ÐØ�>‰>Ò!Ø×Ñ¨x¯©ÐÔ?Ø×*Ñ*Ó,ˆÜ˜h¯©Ó5ˆØ×!×!Ð!Ü*¨8ÀÐTYÔZˆHØ�Š˜8Ÿ?™?Õ*�Ø×Ò˜h×/Ñ/Õ/ÕØ�‰×"Ñ" 8Ö?r5   c                ó¢  € V P                  4        V P                  4        V P                  f.   V P                  '       d   V P	                  V P                  4       V P                  eE   V P                  P                  4        R V n        V'       d   V P                  P                  4        M-V'       d   V P                  P                  4        \        R4      h\        P                  RV P                   RV P                   RV P                   RV P                  '       d   V P                  MR R2	4       V P                  V P                  3# )Nz@Please pass `features` or at least one example when writing datazDone writing Ú z in z bytes ry  Ú.)ru  ro  r  r¹   rI  r)  r   rb   rÌ   Údebugr  r  r  r  )ry   Úclose_streams   &&r3   ÚfinalizeÚArrowWriter.finalize  sü   € Ø×ÑÔ!à×#Ñ#Ô%à�>‰>Ò! d§k§k kØ×Ñ˜tŸ{™{Ô+Ø�>‰>Ò%Ø�N‰N× Ñ Ô"Ø!ˆDŒNßØ—‘×!Ñ!Ô#øçØ—‘×!Ñ!Ô#Ü&Ð'iÓjÐjÜ�‰Ø˜D×.Ñ.Ð/¨q°·±°¸4ÀÇÁÐ?PÐPWÐfj×fp×fpÐfpÐX\×XbÒXbÐvxÐWyÐyzÐ{ô	
ð ×!Ñ! 4§?¡?Ð2Ð2r5   )r  r  r  r  r  r  r  r  r  r  r  r  r  r  rp   r  r   r  r  r  r  )NNNNNNFFrq   TrÚ   FNr©   )NT)T)rd   re   rf   rg   rå   rz   r#  r&  r-  r)  rD  rI  Úpropertyr¹   ræ   r@  ro  ru  r{  r€  r‡  rg  rŽ  rs  r§  rh   rç   rè   s   @r3   rý   rý   ç  sÃ   ø‡ € Ù6÷<ò <ò|Xòòò ÷ ð  ÷6Oð Oð ñ6ó ð6ð ÷5ñ 5ó ð5ò #òD÷*ò *÷$&ò &÷tò t÷")6ò )6÷VIò I÷@ò @÷3ò 3r5   rý   c                   óP   a a€ ] tR tRt oRRRR/V 3R lltV3R lR ltRtVtV ;t# )	ÚParquetWriteri"  Úuse_content_defined_chunkingTÚwrite_page_indexc               ól   <€ \         SV `  ! V/ VB  VR J d   \        P                  pWn        W n        R# )TN)rö   rz   r   ÚDEFAULT_CDC_OPTIONSr¬  r­  )ry   r¬  r­  ÚargsÚkwargsrø   s   &$$*,€r3   rz   ÚParquetWriter.__init__#  s5   ø€ Ü‰Ò˜$Ð) &Ò)Ø'¨4Ó/Ü+1×+EÑ+EÐ(Ø,HÔ)Ø 0Ör5   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r4  r6  )r2   rr   s   "€r3   r4   ÚParquetWriter.__annotate__*  s   ø€ ÷ ñ ©R¯Y©Yñ r5   c                ó  € V P                  V4      w  V n        V n        \        P                  ! V P
                  V P                  V P                  V P                  V P                  P                  4        UUu/ uF  w  r#T\        V4      '       d   R MRbK  	  uppV P                  P                  4        UUu. uF  w  r#\        V4      '       d   K  VNK  	  uppV P                  P                  4        UUu/ uF  w  r#\        V4      '       g   K  VRbK  	  uppR7      V n
        V P                  RJd>   V P                  P                  R\        P                  ! V P                  4      /4       R# R# u uppi u uppi u uppi )ÚnoneÚsnappyÚPLAIN)r¬  r­  ÚcompressionÚuse_dictionaryÚcolumn_encodingFÚcontent_defined_chunkingN)rD  r  r  Úpqr«  r   r¬  r­  Úitemsr   r  Úadd_key_value_metadatarR  rS  )ry   r5  rì   r9   s   &&  r3   rI  ÚParquetWriter._build_writer*  s>  € Ø'+×'9Ñ'9¸/Ó'JÑ$ˆŒ�d”nÜ×)Ò)Ø�K‰KØ�L‰LØ)-×)JÑ)JØ!×2Ñ2à`d×`nÑ`n×`tÑ`tÔ`vôÙ`vÑP\ÐPS�Ô4°W×=Ò=‘VÀ8ÒKÑ`vòð 59·N±N×4HÑ4HÔ4JÔqÑ4J¡L CÔRgÐho×RpŸC˜CÑ4JÒqà15·±×1EÑ1EÔ1GôÙ1G¡ ÔK`Ðah×Ki”��W’Ñ1Gòô
ˆŒð ×,Ñ,°EÓ9Ø�N‰N×1Ñ1Ø+¬T¯ZªZ¸×8YÑ8YÓ-ZÐ[öñ :ùóùó rùós   Á8!E7Â9E=ÃE=Ã9FÄF)r  r  r  r¬  r­  )	rd   re   rf   rg   rz   rI  rh   rç   rú   rû   s   @@r3   r«  r«  "  s(   ù‡ € ð1¸4ð 1ÐRV÷ 1÷÷ ð r5   r«  )Qrå   rÒ   rR  r‰   Úcollections.abcr   Útypingr   r   r   ÚfsspecÚnumpyrK   Úpyarrowr~   Úpyarrow.jsonrÐ   Úpyarrow.parquetÚparquetr½  Úfsspec.corer   ry  r   r.   r   r	   r
   r   r   r   Úfeatures.featuresr   r   r   r   r   r   r   r   r   r   r   Úfilesystemsr   rÍ   r   rt  r   r   r   r   Úutilsr   Ú
utils.jsonr    r!   r"   r#   r$   r%   r&   r'   Úutils.py_utilsr(   r)   r*   Ú
get_loggerrd   rÌ   rm   rÉ   rN   rY   r`   ru   rb   rj   rê   rý   r«  rc   r5   r3   Ú<module>rÐ     sÛ   ðñ +ã 	Û Û 
Ý $ß )Ñ )ã Û Û Ý Ý Ý !å ß ?× ?÷÷ ÷ ñ õ .Ý ß UÓ UÝ ÷	÷ 	ó 	÷ ]Ñ \ð 
×	Ò	˜HÓ	%€à€õ$8õN%8õPwô0	˜:ô 	÷^ñ ^ôB	
˜]ô 
÷2x3ñ x3ôv	�Kö r5   