+
    QV-jb  ã                   ó®  € R t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	H
t
HtHt ^ RIt^ RIHt ^ RIHt ^RIHt ^RIHtHt ^RIHtHtHtHt ^R	IHt ^R
IH t! ]
'       d   ^RI"H#t# ^RI$H%t%H&t& ]PN                  ! ](4      t)Rt*]PV                  ! R]^R+  R2]PX                  4      t-]PV                  ! R4      t. ! R R]/4      t0 ! R R]/4      t1]! RR7       ! R R4      4       t2R,R R llt3 ! R R4      t4 ! R R]44      t5 ! R R ]44      t6]! RR7       ! R! R"4      4       t7]! RR7       ! R# R$4      4       t8R% t9R& t:R' t;R( t< ! R) R*4      t=R# )-zArrow ArrowReader.N)Ú	dataclass)Úpartial)ÚTYPE_CHECKINGÚOptionalÚUnion)Ú
thread_map)ÚDownloadConfig)Ú	_split_reÚfilenames_for_dataset_split)ÚInMemoryTableÚMemoryMappedTableÚTableÚconcat_tables)Úlogging)Útqdm)ÚDatasetInfo)ÚSplitÚ	SplitInfoz=https://storage.googleapis.com/huggingface-nlp/cache/datasetsz
^
 (?P<split>zŒ)
 (\[
    ((?P<from>-?[\d_]+)
     (?P<from_pct>%)?)?
    :
    ((?P<to>-?[\d_]+)
     (?P<to_pct>%)?)?
 \])?(\((?P<rounding>[^\)]*)\))?
$
z\s*\+\s*c                   ó   € ] tR t^AtRtRtR# )ÚDatasetNotOnHfGcsErrorz?When you can't get the dataset from the Hf google cloud storage© N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__static_attributes__r   ó    Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/datasets/arrow_reader.pyr   r   A   s   † ÙIãr   r   c                   ó   € ] tR t^GtRtRtR# )ÚMissingFilesOnHfGcsErrorz9When some files are missing on the Hf oogle cloud storager   Nr   r   r   r   r!   r!   G   s   † ÙCãr   r!   T)Úfrozenc                   ó0   a € ] tR t^Mt o RtV 3R ltRtV tR# )ÚFileInstructionsae  The file instructions associated with a split ReadInstruction.

Attributes:
    num_examples: `int`, The total number of examples
    file_instructions: List[dict(filename, skip, take)], the files information.
        The filenames contains the relative path, not absolute.
        skip/take indicates which example read in the file: `ds.slice(skip, take)`
c                óB   <€ V ^8„  d   Qh/ S[ ;R&   S[S[,          ;R&   # )é   Únum_examplesÚfile_instructions)ÚintÚlistÚdict)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚFileInstructions.__annotate__M   s#   ø‡ ‚ ñ Ññ ñ ™D•zÑ!ò r   r   N©r   r   r   r   r   Ú__annotate_func__r   Ú__classdictcell__©r-   s   @r   r$   r$   M   s   ø‡ € ñ÷ ƒ r   r$   c                ó´   € V ^8„  d   QhR\         R\        R,          R\        \         R3,          R\        \         ,          R\        \         ,          R\        /# )	r&   ÚnameÚsplit_infosr   ÚinstructionÚReadInstructionÚfiletype_suffixÚprefix_pathÚreturn)Ústrr*   r   r   r$   )r,   s   "r   r.   r.   \   sd   € ÷ Hñ HÜ
ðHä�kÕ"ðHô ”sÐ-Ð-Õ.ðHô œc•]ð	Hô
 œ#•ðHô ñHr   c                ó  € \        V \        4      '       g"   \        R\        V 4      P                   24      hV '       g   \        R4      hV Uu/ uF  qUP                  VP                  bK  	  ppV Uu/ uF  qUP                  VP                  bK  	  ppV Uu/ uF8  pVP                  \        VV VP                  VWuP                  ,          R7      bK:  	  pp\        V\        4      '       g   \        P                  V4      pVP                  V4      p	. p
^ pV	 EFP  pWlP                  ,          pWŒP                  ,          pW|P                  ,          pVP                  f   ^ MVP                  pVP                  f   TMVP                  pVf>   V F5  pVV,
          pV^ 8X  d   K  VV,          pV
P!                  RVRVRV/4       K7  	  K³  ^ p^ p\#        Wï4       FŠ  w  ppVV,          pVV8  dl   VV8”  de   VV8”  d
   VV,
          M^ pVV8  d   VV,
          V,
          MRpV^ 8X  d   KQ  V
P!                  RVRVRV/4       TVR8X  d
   VV,
          MT,          pVV,          pKŒ  	  EKS  	  \%        VV
R7      # u upi u upi u upi )	aÃ  Returns instructions of the split dict.

Args:
    name (`str`): Name of the dataset.
    split_infos (`list` of `[SplitInfo]`): Dataset splits information.
    instruction ([`ReadInstruction`] or `str`): Reading instruction for a dataset.
    filetype_suffix (`str`, *optional*): Suffix of dataset files, e.g. 'arrow' or 'parquet'.
    prefix_path (`str`, *optional*): Prefix of dataset files, e.g. directory name.

Returns:
    [`FileInstructions`]
zExpected str 'name', but got: zExpected non-empty str 'name')ÚpathÚdataset_nameÚsplitr9   Úshard_lengthsÚfilenameÚskipÚtake)r'   r(   éÿÿÿÿ)Ú
isinstancer<   Ú	TypeErrorÚtyper   Ú
ValueErrorr5   r'   rA   r
   r8   Ú	from_specÚto_absoluteÚ	splitnameÚfrom_ÚtoÚappendÚzipr$   )r5   r6   r7   r9   r:   ÚinfoÚname2lenÚname2shard_lengthsÚname2filenamesÚabsolute_instructionsr(   r'   Ú	abs_instrÚsplit_lengthÚ	filenamesrA   rM   rN   rB   rD   Úindex_startÚ	index_endÚshard_lengthrC   s   &&&&&                   r   Úmake_file_instructionsr\   \   s„  € ô& �dœC× Ò ÜÐ8¼¸d»×9LÑ9LÐ8MÐNÓOÐOßÜÐ8Ó9Ð9Ù9DÓE¹°—	‘	˜4×,Ñ,Ò,¹€HÐEÙDOÓPÁK¸DŸ)™) T×%7Ñ%7Ò7ÁKÐÐPñ  ó	ñ  ˆDð 	�	‰	Ô.ØØØ—)‘)Ø+Ø,¯Y©YÕ7ô
ò 	
ñ  ð ð 	ô �k¤?×3Ò3Ü%×/Ñ/°Ó<ˆà'×3Ñ3°HÓ=Ðð ÐØ€LÜ*ˆ	Ø× 3Ñ 3Õ4ˆØ"×#6Ñ#6Õ7ˆ	Ø*×+>Ñ+>Õ?ˆØ—_‘_Ò,‘°)·/±/ˆØ&Ÿ\™\Ò1‰\°y·|±|ˆØÒ Û%�Ø˜E•z�Ø˜1”9ÙØ Õ$�Ø!×(Ñ(¨*°hÀÈÈvÐW[Ð)\Ö]ó &ð ˆKØˆIÜ*-¨iÖ*GÑ&�˜,Ø˜\Õ)�	Ø˜9Ô$¨¨kÔ)9Ø27¸+Ô2E˜5 ;Ö.È1�DØ68¸9´n˜2 Õ+¨dÖ2È"�DØ˜q”yÙ Ø%×,Ñ,¨j¸(ÀFÈDÐRXÐZ^Ð-_Ô`Ø ¸4À2¼: L°4Ö$7È4ÕO�LØ˜|Õ+’ô +Hñ! +ô4 Ø!Ø+ôð ùò] FùÚPùò	s   ÁI=Á4JÂ>Jc                   óˆ   a € ] tR t^§t o RtV 3R lR ltRV 3R lR lltRV 3R lR lltR tRR	 lt	RV 3R lR llt
RtV tR
# )Ú
BaseReaderz8
Build a Dataset object out of Instruction instance(s).
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# ©r&   r>   rQ   r   ©r<   r   )r,   r-   s   "€r   r.   ÚBaseReader.__annotate__¬   s    ø€ ÷ 	4ñ 	4™Sð 	4©°Õ(?ñ 	4r   c                ó,   € Wn         W n        RV n        R# )z�Initializes ArrowReader.

Args:
    path (str): path where tfrecords are stored.
    info (DatasetInfo): info about the dataset.
N)Ú_pathÚ_infoÚ_filetype_suffix)Úselfr>   rQ   s   &&&r   Ú__init__ÚBaseReader.__init__¬   s   € ð Œ
Ø.2Œ
Ø/3ˆÖr   c                ó    <€ V ^8„  d   QhRS[ /# ©r&   r;   ©r   )r,   r-   s   "€r   r.   rb   ·   s   ø€ ÷ "ñ "Éuñ "r   c                ó   € \         h)ú=Returns a Dataset instance from given (filename, skip, take).)ÚNotImplementedError)rg   Úfilename_skip_takeÚ	in_memorys   &&&r   Ú_get_table_from_filenameÚ#BaseReader._get_table_from_filename·   s   € ä!Ð!r   c                ó    <€ V ^8„  d   QhRS[ /# rk   rl   )r,   r-   s   "€r   r.   rb   »   s   ø€ ÷ ñ ±Uñ r   c           	     óŒ  € \        V4      ^ 8X  g;   \        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   \        R4      h\        P                  ! V4      pV F7  p\
        P                  P                  V P                  VR,          4      VR&   K9  	  \        \        V P                  VR7      T\        R\        V4      ^8*  ;'       g    RR	7      pV Uu. uF  p\        V4      ^ 8”  g   K  VNK  	  ppV'       g2   V P                  e   V P                  P                  f   \        R
4      hT;'       gL    \        P                   ! . \"        P$                  ! V P                  P                  P&                  4      R7      .p\        V4      ^8w  d   \)        V4      pV# V^ ,          pV# u upi )aS  Returns Dataset for given file instructions.

Args:
    files: List[dict(filename, skip, take)], the files information.
        The filenames contain the absolute path, not relative.
        skip/take indicates which example read in the file: `ds.slice(skip, take)`
    in_memory (bool, default False): Whether to copy the data in-memory.
c              3   óB   "  € T F  p\        V\        4      x € K  	  R # 5i©N)rF   r+   )Ú.0Úfs   & r   Ú	<genexpr>Ú)BaseReader._read_files.<locals>.<genexpr>Ä   s   é € Ð%IÁ5¸a¤j°´D×&9Ð&9Ã5ùs   ‚FTz&please provide valid file informationsrB   ©rq   zLoading dataset shardsN)Ú
tqdm_classÚdescÚdisablezqTried to read an empty table. Please specify at least info.features to create an empty table with the right type.)Úschema)ÚlenÚallrI   ÚcopyÚdeepcopyÚosr>   Újoinrd   r   r   rr   Úhf_tqdmre   Úfeaturesr   Úfrom_batchesÚpar€   rH   r   )rg   Úfilesrq   ry   Ú	pa_tablesÚtÚpa_tables   &&&    r   Ú_read_filesÚBaseReader._read_files»   si  € ô ˆu‹:˜Œ?§#£#Ñ%IÁ5Ó%I§#§#¢#Ñ%IÁ5Ó%I×"IÒ"IÜÐEÓFÐFÜ—’˜eÓ$ˆÛˆAÜŸG™GŸL™L¨¯©°Q°zµ]ÓCˆAˆj‹Mñ ô Ü�D×1Ñ1¸YÔGØÜØ)ä˜“J "Ñ$×,Ð,¨ô
ˆ	ñ !*Ó8¡	˜1¬S°«V°a©Z—Q�Q¡	ˆ	Ð8ß˜dŸj™jÒ0°D·J±J×4GÑ4GÒ4OÜð Dóð ð ×mÐm¤-×"<Ò"<¸RÌÏ	Ê	ÐRV×R\ÑR\×ReÑRe×RjÑRjÓHkÔ"lÐ!mˆ	Ü/2°9«~ÀÔ/B”= Ó+ˆØˆð IRÐRSÍˆØˆùò 9s   Ã,GÄGc                ób   € \        WW P                  V P                  R7      pVP                  pV# )z?Return list of dict {'filename': str, 'skip': int, 'take': int})r9   r:   )r\   rf   rd   r(   )rg   r5   r7   r6   r(   r‹   s   &&&&  r   Úget_file_instructionsÚ BaseReader.get_file_instructionsÛ   s3   € ä2Ø˜{×<QÑ<QÐ_c×_iÑ_iô
Ðð "×3Ñ3ˆØˆr   c                ó~   € V P                  WV4      pV'       g   RV R2p\        V4      hV P                  WRVR7      # )a»  Returns Dataset instance(s).

Args:
    name (str): name of the dataset.
    instructions (ReadInstruction): instructions to read.
        Instruction can be string and will then be passed to the Instruction
        constructor as it.
    split_infos (list of SplitInfo proto): the available splits for dataset.
    in_memory (bool, default False): Whether to copy the data in-memory.

Returns:
     kwargs to build a single Dataset instance.
zInstruction "z" corresponds to no data!)r‹   Úoriginal_instructionsrq   )r’   rI   Ú
read_files)rg   r5   Úinstructionsr6   rq   r‹   Úmsgs   &&&&&  r   ÚreadÚBaseReader.readã   sF   € ð* ×*Ñ*¨4¸{ÓKˆßØ! , Ð/HÐIˆCÜ˜S“/Ð!Ø�‰ UÐZcˆÓdÐdr   Nc                óD   <€ V ^8„  d   QhRS[ S[,          RS[R,          /# )r&   r‹   r•   )Nr8   r   )r*   r+   r   )r,   r-   s   "€r   r.   rb   þ   s)   ø€ ÷ ñ á‘D�zðñ  %Ð%EÕFñr   c                óŠ   € V P                  WR7      pVe   ^RIHp V! \        V4      4      pMRpRVRV P                  RV/pV# )a  Returns single Dataset instance for the set of file instructions.

Args:
    files: List[dict(filename, skip, take)], the files information.
        The filenames contains the relative path, not absolute.
        skip/take indicates which example read in the file: `ds.skip().take()`
    original_instructions: store the original instructions used to build the dataset split in the dataset.
    in_memory (bool, default False): Whether to copy the data in-memory.

Returns:
    kwargs to build a Dataset instance.
r|   N)r   Úarrow_tablerQ   r@   )r�   Úsplitsr   r<   re   )rg   r‹   r•   rq   rŽ   r   r@   Údataset_kwargss   &&&&    r   r–   ÚBaseReader.read_filesþ   sR   € ð& ×#Ñ# EÐ#Ó?ˆà Ò,Ý%áœ#Ð3Ó4Ó5‰EàˆEØ'¨°6¸4¿:¹:ÀwÐPUÐVˆØÐr   )rf   re   rd   ©F)NF)r   r   r   r   r   rh   rr   r�   r’   r™   r–   r   r2   r3   s   @r   r^   r^   §   sC   ø‡ € ñ÷	4ð 	4÷"ò "÷ò ò@ôe÷6÷ ð r   r^   c                   óx   a a€ ] tR tRt oRtV3R lV 3R lltR
V3R lR llt]R
V3R lR ll4       tR	t	Vt
V ;t# )ÚArrowReaderi  z�
Build a Dataset object out of Instruction instance(s).
This Reader uses either memory mapping or file descriptors (in-memory) on arrow files.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r`   ra   )r,   r-   s   "€r   r.   ÚArrowReader.__annotate__#  s    ø€ ÷ (ñ (™Sð (©°Õ(?ñ (r   c                ó4   <€ \         SV `  W4       RV n        R# )zƒInitializes ArrowReader.

Args:
    path (str): path where Arrow files are stored.
    info (DatasetInfo): info about the dataset.
ÚarrowN©Úsuperrh   rf   ©rg   r>   rQ   Ú	__class__s   &&&€r   rh   ÚArrowReader.__init__#  s   ø€ ô 	‰Ñ˜Ô$Ø 'ˆÖr   c                ó    <€ V ^8„  d   QhRS[ /# rk   rl   )r,   r-   s   "€r   r.   r¥   -  s   ø€ ÷ ñ Éuñ r   c                ó  € VR,          RV9   d
   VR,          MRRV9   d
   VR,          MRrTp\         P                  W2R7      pVR8X  d   \        V4      V,
          pVe-   Ve)   V^ 8X  d   V\        V4      8X  g   VP                  WE4      pV# )rn   rB   rC   NrD   r|   rE   )r£   Ú
read_tabler�   Úslice)rg   rp   rq   rB   rC   rD   Útables   &&&    r   rr   Ú$ArrowReader._get_table_from_filename-  s•   € ð ˜zÕ*Ø*0Ð4FÔ*FÐ˜vÖ&ÈDØ*0Ð4FÔ*FÐ˜vÖ&ÈDð ˆô
 ×&Ñ& xÐ&ÓEˆØ�2Œ:Ü�u“: Õ$ˆDàÒ Ò 0¸$À!¼)ÈÔPSÐTYÓPZÔHZØ—K‘K Ó+ˆEØˆr   c                ó    <€ V ^8„  d   QhRS[ /# rk   rl   )r,   r-   s   "€r   r.   r¥   =  s   ø€ ÷ -ñ -±ñ -r   c                óL   € V'       d   \         M\        pVP                  V 4      # )z¯
Read table from file.

Args:
    filename (str): File name of the table.
    in_memory (bool, default=False): Whether to copy the data in-memory.

Returns:
    pyarrow.Table
)r   r   Ú	from_file)rB   rq   Ú	table_clss   && r   r¯   ÚArrowReader.read_table<  s   € ÷ &/•MÔ4Eˆ	Ø×"Ñ" 8Ó,Ð,r   ©rf   r¡   )r   r   r   r   r   rh   rr   Ústaticmethodr¯   r   r2   Ú__classcell__©r«   r-   s   @@r   r£   r£     s<   ù‡ € ñ÷
(ó (÷ò ð ÷-ñ -ó ÷-ð -r   r£   c                   óH   a a€ ] tR tRt oRtV3R lV 3R lltR tRtVtV ;t	# )ÚParquetReaderiL  zj
Build a Dataset object out of Instruction instance(s).
This Reader uses memory mapping on parquet files.
c                ó4   <€ V ^8„  d   QhRS[ RS[R,          /# r`   ra   )r,   r-   s   "€r   r.   ÚParquetReader.__annotate__R  s    ø€ ÷ *ñ *™Sð *©°Õ(?ñ *r   c                ó4   <€ \         SV `  W4       RV n        R# )zƒInitializes ParquetReader.

Args:
    path (str): path where tfrecords are stored.
    info (DatasetInfo): info about the dataset.
ÚparquetNr¨   rª   s   &&&€r   rh   ÚParquetReader.__init__R  s   ø€ ô 	‰Ñ˜Ô$Ø )ˆÖr   c                óî   € VR,          RV9   d
   VR,          MRRV9   d
   VR,          MRrTp\         P                  ! VRR7      pVe-   Ve)   V^ 8X  d   V\        V4      8X  g   VP                  WE4      pV# )rn   rB   rC   NrD   T)Ú
memory_map)Úpqr¯   r�   r°   )rg   rp   ÚkwargsrB   rC   rD   rŽ   s   &&,    r   rr   Ú&ParquetReader._get_table_from_filename\  s}   € ð ˜zÕ*Ø*0Ð4FÔ*FÐ˜vÖ&ÈDØ*0Ð4FÔ*FÐ˜vÖ&ÈDð ˆô —=’= °dÔ;ˆàÒ Ò 0¸$À!¼)ÈÔPSÐT\ÓP]ÔH]Ø—~‘~ dÓ1ˆHØˆr   r¸   )
r   r   r   r   r   rh   rr   r   r2   rº   r»   s   @@r   r½   r½   L  s   ù‡ € ñ÷
*ó *÷ò r   r½   c                   ó0   a € ] tR tRt o RtV 3R ltRtV tR# )Ú_AbsoluteInstructionik  z?A machine friendly slice: defined absolute positive boundaries.c                ó>   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   # )r&   rL   rM   rN   )r<   r)   )r,   r-   s   "€r   r.   Ú!_AbsoluteInstruction.__annotate__k  s)   ø‡ ‚ ñ �Nñ	 ñ
 �Jñ ñ 	�Gò r   r   Nr0   r3   s   @r   rÉ   rÉ   k  s   ø‡ € áI÷ ƒ r   rÉ   c                   óF   a € ] tR tRt o RtRtRtRtRtR t	V 3R lt
RtV tR# )Ú_RelativeInstructionit  zHRepresents a single parsed slicing instruction, can use % and negatives.Nc                ó†  € V P                   e   V P                   R9  d   \        R4      hV P                  e   V P                  R	9  d   \        R4      hV P                   R8w  d   V P                  e   \        R4      hV P                   R8X  d4   V P                  e&   \	        V P                  4      ^d8”  d   \        R4      hV P                   R8X  d4   V P
                  e&   \	        V P
                  4      ^d8”  d   \        R4      hV P                  f   V P                   R8X  d   RMV P                  V P                  R&   R # )
NÚ%zunit must be either % or absÚclosestz5rounding must be either closest or pct1_dropremainderzAIt is forbidden to specify rounding if not using percent slicing.z2Percent slice boundaries must be > -100 and < 100.Úrounding)rÏ   Úabs)rÐ   Úpct1_dropremainder)ÚunitrI   rÑ   rM   rÒ   rN   Ú__dict__©rg   s   &r   Ú__post_init__Ú"_RelativeInstruction.__post_init__~  sò   € Ø�9‰9Ò  T§Y¡Y°lÔ%BÜÐ;Ó<Ð<Ø�=‰=Ò$¨¯©Ð>_Ô)_ÜÐTÓUÐUØ�9‰9˜Ô §¡Ò 9ÜÐ`ÓaÐaØ�9‰9˜Ô §
¡
Ò 6¼3¸t¿z¹z»?ÈSÔ;PÜÐQÓRÐRØ�9‰9˜Ô §¡Ò 3¼¸D¿G¹G»ÀsÔ8JÜÐQÓRÐRà15·±Ò1FÈ4Ï9É9ÐX[ÔK[¡IÐae×anÑanˆ�‰�jÓ!r   c                ó–   <€ V ^8„  d   Qh/ S[ ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[ ,          ;R&   S[S[ ,          ;R&   # )r&   rL   rM   rN   rÔ   rÑ   )r<   r   r)   )r,   r-   s   "€r   r.   Ú!_RelativeInstruction.__annotate__t  sU   ø‡ ‚ ñ �Nñ	 ñ
 ‘C�=Ññ ñ 	‘�Ññ ñ ‘3�-Ññ ñ ‘s�mÑ"ò r   r   )r   r   r   r   r   rM   rN   rÔ   rÑ   r×   r1   r   r2   r3   s   @r   rÍ   rÍ   t  s+   ø‡ € áRð  €EØ€BØ€DØ"€Hòo÷ ƒ r   rÍ   c           
     óè  € \         P                  V 4      pV'       g   \        RV  24      hVP                  R4      '       g   VP                  R4      '       d   RMRp\	        VP                  R4      VP                  R4      VP                  R4      '       d   \        VP                  R4      4      MR	VP                  R
4      '       d!   \        VP                  R
4      4      VR7      # R	VR7      # )z)Returns ReadInstruction for given string.z!Unrecognized instruction format: Úfrom_pctÚto_pctrÏ   rÒ   r@   rÑ   ÚfromNrN   )Ú
split_namerÑ   rM   rN   rÔ   )Ú_SUB_SPEC_REÚmatchrI   Úgroupr8   r)   )ÚspecÚresrÔ   s   &  r   Ú_str_to_read_instructionrå   �  sÂ   € ä
×
Ñ
˜TÓ
"€CßÜÐ<¸T¸FÐCÓDÐDØ—)‘)˜J×'Ò'¨3¯9©9°X×+>Ò+>‰3ÀE€DÜØ—9‘9˜WÓ%Ø—‘˜:Ó&Ø(+¯	©	°&×(9Ò(9Œc�#—)‘)˜FÓ#Ô$¸tØ#&§9¡9¨T§?¢?Œ3ˆs�y‰y˜‹ÓØôð ð 9=Øôð r   c                 ór   € V^d8  d   Rp\        V4      hV \        P                  ! VR,          4      ,          # )éd   zUsing "pct1_dropremainder" rounding on a split with less than 100 elements is forbidden: it always results in an empty dataset.ç      Y@)rI   ÚmathÚtrunc)Úboundaryr'   r˜   s   && r   Ú_pct_to_abs_pct1rì   œ  s:   € à�cÔðLð 	ô ˜‹oÐØ”d—j’j °Õ!5Ó6Õ6Ð6r   c                 óD   € \        \        W,          R ,          4      4      # )rè   )r)   Úround)rë   r'   s   &&r   Ú_pct_to_abs_closestrï   §  s   € ÜŒu�XÕ,¨uÕ4Ó5Ó6Ð6r   c                óú  € V P                   R8X  d   \        M\        pV P                  pW19  d   \	        RV R\        V4       R24      hW,          pV P                  pV P                  pV P                  R8X  d   Vf   ^ MV! WT4      pVf   TMV! Wd4      pMVf   ^ MTpVf   TMTpV^ 8  d   \        WE,           ^ 4      pV^ 8  d   \        WF,           ^ 4      p\        WT4      p\        Wd4      p\        W5V4      # )z¨Returns _AbsoluteInstruction instance for given RelativeInstruction.

Args:
    rel_instr: RelativeInstruction instance.
    name2len: dict {split_name: num_examples}.
rÐ   zUnknown split "z". Should be one of Ú.rÏ   )rÑ   rï   rì   rL   rI   r*   rM   rN   rÔ   ÚmaxÚminrÉ   )Ú	rel_instrrR   Ú
pct_to_absr@   r'   rM   rN   s   &&     r   Ú_rel_to_abs_instrrö   «  sô   € ð )2×(:Ñ(:¸iÔ(GÕ$ÔM]€JØ×Ñ€EØÔÜ˜?¨5¨'Ð1EÄdÈ8ÃnÐEUÐUVÐWÓXÐXØ•?€LØ�O‰O€EØ	�‰€BØ‡~�~˜ÔØ’]‘©
°5Ó(GˆØšZ‰\©Z¸Ó-I‰à’]‘¨ˆØšZ‰\¨RˆØˆq„yÜ�LÕ(¨!Ó,ˆØ	ˆA„vÜ�Õ" AÓ&ˆÜ�Ó$€EÜ	ˆRÓ	€BÜ ¨bÓ1Ð1r   c                   ór   a € ] tR tRt o RtR t]R 4       tRR lt]R 4       t	R t
R	 tR
 tR tR tRtV tR# )r8   iÈ  a—  Reading instruction for a dataset.

Examples::

  # The following lines are equivalent:
  ds = datasets.load_dataset('ylecun/mnist', split='test[:33%]')
  ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec('test[:33%]'))
  ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction('test', to=33, unit='%'))
  ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction(
      'test', from_=0, to=33, unit='%'))

  # The following lines are equivalent:
  ds = datasets.load_dataset('ylecun/mnist', split='test[:33%]+train[1:-1]')
  ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec(
      'test[:33%]+train[1:-1]'))
  ds = datasets.load_dataset('ylecun/mnist', split=(
      datasets.ReadInstruction('test', to=33, unit='%') +
      datasets.ReadInstruction('train', from_=1, to=-1, unit='abs')))

  # The following lines are equivalent:
  ds = datasets.load_dataset('ylecun/mnist', split='test[:33%](pct1_dropremainder)')
  ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction.from_spec(
      'test[:33%](pct1_dropremainder)'))
  ds = datasets.load_dataset('ylecun/mnist', split=datasets.ReadInstruction(
      'test', from_=0, to=33, unit='%', rounding="pct1_dropremainder"))

  # 10-fold validation:
  tests = datasets.load_dataset(
      'ylecun/mnist',
      [datasets.ReadInstruction('train', from_=k, to=k+10, unit='%')
      for k in range(0, 100, 10)])
  trains = datasets.load_dataset(
      'ylecun/mnist',
      [datasets.ReadInstruction('train', to=k, unit='%') + datasets.ReadInstruction('train', from_=k+10, unit='%')
      for k in range(0, 100, 10)])

c                ó   € Wn         R # rw   ©Ú_relative_instructions)rg   Úrelative_instructionss   &&r   Ú_initÚReadInstruction._initï  s   € à&;Ö#r   c                óJ   € V P                  V 4      pVP                  V4       V# )zCReturns ReadInstruction obj initialized with relative_instructions.)Ú__new__rü   )Úclsrû   Úresults   && r   Ú,_read_instruction_from_relative_instructionsÚ<ReadInstruction._read_instruction_from_relative_instructionsó  s$   € ð —‘˜SÓ!ˆØ�‰Ð*Ô+Øˆr   Nc           	     ó@   € V P                  \        WWEV4      .4       R# )aq  Initialize ReadInstruction.

Args:
    split_name (str): name of the split to read. Eg: 'train'.
    rounding (str, optional): The rounding behaviour to use when percent slicing is
        used. Ignored when slicing with absolute indices.
        Possible values:
         - 'closest' (default): The specified percentages are rounded to the
             closest value. Use this if you want specified percents to be as
             much exact as possible.
         - 'pct1_dropremainder': the specified percentages are treated as
             multiple of 1%. Use this option if you want consistency. Eg:
                 len(5%) == 5 * len(1%).
             Using this option, one might not be able to use the full set of
             examples, if the number of those is not a multiple of 100.
    from_ (int):
    to (int): alternative way of specifying slicing boundaries. If any of
        {from_, to, unit} argument is used, slicing cannot be specified as
        string.
    unit (str): optional, one of:
        '%': to set the slicing unit as percents of the split size.
        'abs': to set the slicing unit as absolute numbers.
N)rü   rÍ   )rg   rß   rÑ   rM   rN   rÔ   s   &&&&&&r   rh   ÚReadInstruction.__init__û  s   € ð6 	�
‰
Ô(¨¸BÀhÓOÐPÖQr   c                óÆ   € \        V4      p\        P                  V4      pV'       g   \        RV 24      h\	        V^ ,          4      p\        R VR,           4       V4      # )aÅ  Creates a `ReadInstruction` instance out of a string spec.

Args:
    spec (`str`):
        Split(s) + optional slice(s) to read + optional rounding
        if percents are used as the slicing unit. A slice can be specified,
        using absolute numbers (`int`) or percentages (`int`).

Examples:

    ```
    test: test split.
    test + validation: test split + validation split.
    test[10:]: test split, minus its first 10 records.
    test[:10%]: first 10% records of test split.
    test[:20%](pct1_dropremainder): first 10% records, rounded with the pct1_dropremainder rounding.
    test[:-5%]+train[40%:60%]: first 95% of test + middle 20% of train.
    ```

Returns:
    ReadInstruction instance.
z&No instructions could be built out of c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irw   )rå   )rx   Úsubs   & r   rz   Ú,ReadInstruction.from_spec.<locals>.<genexpr>5  s   é € ÐF¹X°cÔ,¨S×1Ð1»Xùs   ‚:é   NN)r<   Ú_ADDITION_SEP_REr@   rI   rå   Úsum)r   rã   Úsubsr7   s   &&  r   rJ   ÚReadInstruction.from_spec  sX   € ô0 �4‹yˆÜ×%Ñ% dÓ+ˆßÜÐEÀdÀVÐLÓMÐMÜ.¨t°A­wÓ7ˆÜÑF¸TÀ"¾XÓFÈÓTÐTr   c                óú  € . pV P                    FÙ  pVP                  pVP                  f   VP                  ež   VP                  pVP                  pVP                  pVP
                  pVR8X  d   TMRpVe   \        V4      V,           MRpVe   \        V4      V,           MRpRV RV R2pVR8X  d   Ve   VR8w  d   RV R2MRp	W8V	,           ,          pVP                  V4       KÛ  	  R	P                  V4      # )
NrÏ   Ú Ú[Ú:Ú]rÐ   Ú(Ú)Ú+)	rú   rL   rM   rN   rÔ   rÑ   r<   rO   r†   )
rg   Úrel_instr_specsrô   Úrel_instr_specrM   rN   rÔ   rÑ   Ú	slice_strÚrounding_strs
   &         r   Úto_specÚReadInstruction.to_spec7  sù   € ØˆØ×4Ô4ˆIØ&×0Ñ0ˆNØ�‰Ò*¨i¯l©lÒ.FØ!Ÿ™�Ø—\‘\�Ø —~‘~�Ø$×-Ñ-�Ø# sœ{‘t°�Ø-2Ò->œ˜E›
 TÖ)ÀB�Ø')¢~”S˜“W˜t–^¸2�Ø ˜w a¨ t¨1Ð-�	à'+¨s¤{°xÒ7KÐPXÐ\eÔPe�a˜�z ‘OÐkmð ð ¨lÕ":Õ:�Ø×"Ñ" >Ö2ñ 5ð  �x‰x˜Ó(Ð(r   c                ó‚  € \        V\        4      '       g   Rp\        V4      hV P                  pVP                  pV^ ,          P                  R8w  dW   V^ ,          P                  R8w  d?   V P                  ^ ,          P
                  V^ ,          P
                  8w  d   \        R4      hV P                  W4,           4      # )zEReturns a new ReadInstruction obj, result of appending other to self.zAReadInstruction can only be added to another ReadInstruction obj.rÒ   zPIt is forbidden to sum ReadInstruction instances with different rounding values.)rF   r8   rG   rú   rÔ   rÑ   rI   r  )rg   Úotherr˜   Úself_risÚ	other_riss   &&   r   Ú__add__ÚReadInstruction.__add__K  s›   € ä˜%¤×1Ò1ØUˆCÜ˜C“.Ð Ø×.Ñ.ˆØ×0Ñ0ˆ	à�Q�K×Ñ Ô%Ø˜!•×!Ñ! UÔ*Ø×+Ñ+¨AÕ.×7Ñ7¸9ÀQ½<×;PÑ;PÔPäÐoÓpÐpØ×@Ñ@ÀÕAUÓVÐVr   c                ó"   € V P                  4       # rw   )r  rÖ   s   &r   Ú__str__ÚReadInstruction.__str__Z  s   € Ø�|‰|‹~Ðr   c                ó"   € R V P                    R2# )zReadInstruction(r  rù   rÖ   s   &r   Ú__repr__ÚReadInstruction.__repr__]  s   € Ø! $×"=Ñ"=Ð!>¸aÐ@Ð@r   c                óV   € V P                    Uu. uF  p\        W!4      NK  	  up# u upi )a"  Translate instruction into a list of absolute instructions.

Those absolute instructions are then to be added together.

Args:
    name2len (`dict`):
        Associating split names to number of examples.

Returns:
    list of _AbsoluteInstruction instances (corresponds to the + in spec).
)rú   rö   )rg   rR   rô   s   && r   rK   ÚReadInstruction.to_absolute`  s,   € ð IM×HcÒHcÓdÑHc¸9Ô! )Ö6ÑHcÑdÐdùÒds   �&rù   )NNNN)r   r   r   r   r   rü   Úclassmethodr  rh   rJ   r  r!  r$  r'  rK   r   r2   r3   s   @r   r8   r8   È  sb   ø‡ € ñ$òL<ð ñó ðôRð: ñUó ðUò<)ò(WòòA÷eð er   r8   rE   )NN)>r   rƒ   ré   r…   ÚreÚdataclassesr   Ú	functoolsr   Útypingr   r   r   ÚpyarrowrŠ   Úpyarrow.parquetrÁ   rÅ   Útqdm.contrib.concurrentr   Údownload.download_configr   Únamingr	   r
   r±   r   r   r   r   Úutilsr   r   r‡   rQ   r   rž   r   r   Ú
get_loggerr   ÚloggerÚHF_GCP_BASE_URLÚcompileÚXrà   r  ÚConnectionErrorr   r!   r$   r\   r^   r£   r½   rÉ   rÍ   rå   rì   rï   rö   r8   r   r   r   Ú<module>r<     sf  ðñ  ã Û Û 	Û 	Ý !Ý ß 1Ñ 1ã Ý Ý .å 4ß :ß IÓ IÝ Ý "÷ Ý!ß(ð 
×	Ò	˜HÓ	%€àQ€à�zŠzð
à�a˜ˆOÐð 	ðð ‡D�Dó€ð  —:’:˜kÓ*Ð ô	˜_ô 	ô	˜ô 	ñ �$Ô÷"ð "ó ð"÷H÷Vsñ sôl,-�*ô ,-ô^�Jô ñ> �$Ô÷ð ó ðñ �$Ô÷oð oó ðoò0ò7ò7ò2÷:deó der   