Ë
    #täiº"  ã                   óô   — d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZ d„ Zda G d„ d	e«      Z ed
g d¢«       G d„ dej,                  «      «       Z G d„ de«      Zy)é    N)Úsuppress)Úcached_propertyÚwraps)Úparse_qs)ÚAbstractFileSystem)Ú"get_package_version_without_importÚinfer_storage_optionsÚmirror_fromÚtokenizec                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                  óÎ   •— 	  ‰| i |¤ŽS # t         $ rP}|j                  s‚ |j                  ^}} t        |t        «      rd|v rt	        t
        j                  |«      |‚‚ d }~ww xY w)Nzdoes not exist)ÚOSErrorÚargsÚ
isinstanceÚstrÚFileNotFoundErrorÚerrnoÚENOENT)r   ÚkwargsÚ	exceptionÚmessageÚfuncs       €úk/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/fsspec/implementations/arrow.pyÚwrapperz wrap_exceptions.<locals>.wrapper   sg   ø€ ð
	Ù˜Ð( Ñ(Ð(øÜò 	Ø—>’>Øà&Ÿ^™^ˆNˆG�dÜ˜'¤3Ô'Ð,<ÀÑ,GÜ'¬¯©°gÓ>ÀIÐMàûð	ús   ƒ ‹	A$”AAÁA$)r   )r   r   s   ` r   Úwrap_exceptionsr      s    ø€ Ü
ˆ4ƒ[óó ðð €Nó    c                   ó  ‡ — e Zd ZdZdZˆ fd„Zed„ «       Zed„ «       Z	e
d„ «       Zdd„Zd„ Zd	„ Zd
„ Zed„ «       Zed„ «       Zed„ «       Zedd„«       Zedd„«       Zedd„«       Zedd„«       Zed„ «       Zed„ «       Zdˆ fd„	Zˆ fd„Zˆ xZS )ÚArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem.

    Parameters
    ----------
    fs : pyarrow.fs.FileSystem

    Ú/c                 óH   •— t        d«      a|| _        t        ‰| �  di |¤Ž y )NÚpyarrow© )r   ÚPYARROW_VERSIONÚfsÚsuperÚ__init__)Úselfr$   r   Ú	__class__s      €r   r&   zArrowFSWrapper.__init__3   s$   ø€ ä<¸YÓGˆØˆŒÜ‰ÑÑ"˜6Ó"r   c                 ó.   — | j                   j                  S ©N)r$   Ú	type_name©r'   s    r   ÚprotocolzArrowFSWrapper.protocol9   s   € à�w‰w× Ñ Ð r   c                 óp   — dt        | j                  j                  | j                  j                  «      z   S )NÚhdfs_)r   r$   ÚhostÚportr,   s    r   ÚfsidzArrowFSWrapper.fsid=   s$   € àœ $§'¡'§,¡,°·±·±Ó=Ñ=Ð=r   c                 óR   — t        |«      }|d   }|j                  d«      r|dd  }|S )NÚpathz//é   )r	   Ú
startswith)Úclsr4   Úopss      r   Ú_strip_protocolzArrowFSWrapper._strip_protocolA   s1   € ä# DÓ)ˆØ�6‰{ˆØ�?‰?˜4Ô à˜˜�8ˆDØˆr   c                 ó@  — | j                  |«      }ddlm} 	 | j                  j	                   ||«      «      D �cg c]  }| j                  |«      ‘Œ }}|r|S |D �cg c]  }|d   ‘Œ	 c}S c c}w # t        t        f$ r  | j                  |fi |¤Žg}Y ŒAw xY wc c}w )Nr   )ÚFileSelectorÚname)	r9   Ú
pyarrow.fsr;   r$   Úget_file_infoÚ_make_entryr   ÚNotADirectoryErrorÚinfo)r'   r4   Údetailr   r;   ÚentryÚentriess          r   ÚlszArrowFSWrapper.lsJ   s¶   € Ø×#Ñ# DÓ)ˆÝ+ð	2ð "ŸW™W×2Ñ2±<ÀÓ3EÔFóáF�Eð × Ñ  Õ'ØFð ð ñ ØˆNá/6Ó7©w e�E˜&“M¨wÑ7Ð7ùòøô "Ô#5Ð6ò 	2Ø �t—y‘y Ñ0¨Ñ0Ð1ŠGð	2üò
 8s(   ™#A2 ¼A-ÁA2 ÁBÁ-A2 Á2#BÂBc                 ó‚   — | j                  |«      }| j                  j                  |g«      \  }| j                  |«      S r*   )r9   r$   r>   r?   )r'   r4   r   rA   s       r   rA   zArrowFSWrapper.infoZ   s;   € Ø×#Ñ# DÓ)ˆØ—‘×&Ñ&¨ vÓ.‰ˆØ×Ñ Ó%Ð%r   c                 óh   — | j                  |«      }	 | j                  |«       y# t        $ r Y yw xY w)NTF)r9   rA   r   ©r'   r4   s     r   ÚexistszArrowFSWrapper.exists_   s;   € Ø×#Ñ# DÓ)ˆð	Ø�I‰I�dŒOð øô !ò 	Ùð	ús   “% ¥	1°1c                 ó„  — ddl m} |j                  |j                  u rd}n{|j                  |j                  u rd}n`|j                  |j
                  u rFt        t        j                  t        j                  t        j                  «      |j                  «      ‚d}|j                  |j                  ||j                  dœS )Nr   )ÚFileTypeÚ	directoryÚfileÚother)r<   ÚsizeÚtypeÚmtime)r=   rK   rP   Ú	DirectoryÚFileÚNotFoundr   r   r   ÚosÚstrerrorr4   rO   rQ   )r'   rA   rK   Úkinds       r   r?   zArrowFSWrapper._make_entryh   s�   € Ý'à�9‰9˜×*Ñ*Ñ*Ø‰DØ�Y‰Y˜(Ÿ-™-Ñ'Ø‰DØ�Y‰Y˜(×+Ñ+Ñ+Ü#¤E§L¡L´"·+±+¼e¿l¹lÓ2KÈTÏYÉYÓWÐWàˆDð —I‘IØ—I‘IØØ—Z‘Zñ	
ð 	
r   c                 óV  — | j                  |«      j                  d«      }| j                  |«      j                  d«      }| j                  |d«      5 }|› dt        j                  d«      › �}	 | j                  |d«      5 }t        j                  ||«       d d d «       | j                  j                  ||«       	 d d d «       y # 1 sw Y   Œ/xY w# t        $ rA t        t        «      5  | j                  j                  |«       d d d «       ‚ # 1 sw Y   ‚ xY ww xY w# 1 sw Y   y xY w)Nr   Úrbz.tmp.é   Úwb)r9   ÚrstripÚ_openÚsecretsÚ	token_hexÚopenÚshutilÚcopyfileobjr$   ÚmoveÚBaseExceptionr   r   Údelete_file)r'   Úpath1Úpath2r   ÚlstreamÚ	tmp_fnameÚrstreams          r   Úcp_filezArrowFSWrapper.cp_file{   sö   € à×$Ñ$ UÓ+×2Ñ2°3Ó7ˆØ×$Ñ$ UÓ+×2Ñ2°3Ó7ˆà�Z‰Z˜˜tÔ$¨Ø ˜' ¤w×'8Ñ'8¸Ó';Ð&<Ð=ˆIðØ—Y‘Y˜y¨$Ô/°7Ü×&Ñ& w°Ô8÷ 0à—‘—‘˜Y¨Õ.÷ %Ð$÷ 0Ð/ûô !ò ÜÔ/Õ0Ø—G‘G×'Ñ'¨	Ô2÷ 1à÷ 1àúðú÷ %Ð$úsN   ÁDÁ/CÂCÂ$CÃC	ÃCÃDÃ*DÄ	DÄDÄDÄDÄD(c                 ó¼   — | j                  |«      j                  d«      }| j                  |«      j                  d«      }| j                  j                  ||«       y )Nr   )r9   r\   r$   rc   )r'   rf   rg   r   s       r   ÚmvzArrowFSWrapper.mv‹   sJ   € à×$Ñ$ UÓ+×2Ñ2°3Ó7ˆØ×$Ñ$ UÓ+×2Ñ2°3Ó7ˆØ�‰�‰�U˜EÕ"r   c                 ó\   — | j                  |«      }| j                  j                  |«       y r*   )r9   r$   re   rH   s     r   Úrm_filezArrowFSWrapper.rm_file‘   s$   € à×#Ñ# DÓ)ˆØ�‰×Ñ˜DÕ!r   c                 óî   — | j                  |«      j                  d«      }| j                  |«      r)|r| j                  j	                  |«       y t        d«      ‚| j                  j                  |«       y )Nr   z0Can't delete directories without recursive=False)r9   r\   Úisdirr$   Ú
delete_dirÚ
ValueErrorre   )r'   r4   Ú	recursiveÚmaxdepths       r   ÚrmzArrowFSWrapper.rm–   s\   € à×#Ñ# DÓ)×0Ñ0°Ó5ˆØ�:‰:�dÔÙØ—‘×"Ñ" 4Õ(ä Ð!SÓTÐTà�G‰G×Ñ Õ%r   c                 óŽ  — |dk(  r0|r| j                   j                  }n]| j                   j                  }nF|dk(  r| j                   j                  }n*|dk(  r| j                   j                  }nt        d|›�«      ‚i }|dk7  s|s)t        t        j                  d«      d   «      dk\  rd |d<    ||fi |¤Ž}t        | ||||fi |¤ŽS )	NrY   r[   Úabz'unsupported mode for Arrow filesystem: Ú.r   é   Úcompression)
r$   Úopen_input_fileÚopen_input_streamÚopen_output_streamÚopen_append_streamrs   Úintr#   ÚsplitÚ	ArrowFile)	r'   r4   ÚmodeÚ
block_sizeÚseekabler   ÚmethodÚ_kwargsÚstreams	            r   r]   zArrowFSWrapper._open¡   sÅ   € à�4Š<ÙØŸ™×0Ñ0‘àŸ™×2Ñ2‘Ø�TŠ\Ø—W‘W×/Ñ/‰FØ�TŠ\Ø—W‘W×/Ñ/‰FäÐFÀtÀhÐOÓPÐPàˆØ�4Š<™xÜ”?×(Ñ(¨Ó-¨aÑ0Ó1°QÒ6à)-�˜Ñ&Ù˜Ñ( Ñ(ˆä˜˜v t¨T°:ÑHÀÑHÐHr   c                 óŒ   — | j                  |«      }|r| j                  |d¬«       y | j                  j                  |d¬«       y )NT)Úexist_okF©rt   )r9   Úmakedirsr$   Ú
create_dir)r'   r4   Úcreate_parentsr   s       r   ÚmkdirzArrowFSWrapper.mkdir¸   s<   € à×#Ñ# DÓ)ˆÙØ�M‰M˜$¨ˆMÕ.à�G‰G×Ñ˜t¨uÐÕ5r   c                 ó`   — | j                  |«      }| j                  j                  |d¬«       y )NTr‹   )r9   r$   r�   )r'   r4   rŠ   s      r   rŒ   zArrowFSWrapper.makedirsÀ   s)   € à×#Ñ# DÓ)ˆØ�‰×Ñ˜4¨4ÐÕ0r   c                 ó\   — | j                  |«      }| j                  j                  |«       y r*   )r9   r$   rr   rH   s     r   ÚrmdirzArrowFSWrapper.rmdirÅ   s$   € à×#Ñ# DÓ)ˆØ�‰×Ñ˜4Õ r   c                 ón   — | j                  |«      }| j                  j                  |«      j                  S r*   )r9   r$   r>   rQ   rH   s     r   ÚmodifiedzArrowFSWrapper.modifiedÊ   s-   € à×#Ñ# DÓ)ˆØ�w‰w×$Ñ$ TÓ*×0Ñ0Ð0r   c                 óR   •— |j                  d|dv«       t        ‰| �  |fd d dœ|¤ŽS )Nr…   )Nr   )ÚstartÚend)Ú
setdefaultr%   Úcat_file)r'   r4   r–   r—   r   r(   s        €r   r™   zArrowFSWrapper.cat_fileÏ   s4   ø€ Ø×Ñ˜* e°9Ð&<Ô=Ü‰wÑ ÐE¨D°dÑE¸fÑEÐEr   c                 óL   •— |j                  dd«       t        ‰| �  ||fi |¤Ž y )Nr…   F)r˜   r%   Úget_file)r'   ÚrpathÚlpathr   r(   s       €r   r›   zArrowFSWrapper.get_fileÓ   s'   ø€ Ø×Ñ˜* eÔ,Ü‰Ñ˜ Ñ0¨Ó0r   )F)FN)rY   NT)T)NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úroot_markerr&   Úpropertyr-   r   r2   Úclassmethodr9   rE   rA   rI   r?   r   rk   rm   ro   rv   r]   r�   rŒ   r’   r”   r™   r›   Ú__classcell__©r(   s   @r   r   r   (   s*  ø„ ñð €Kô#ð ñ!ó ð!ð ñ>ó ð>ð ñó ðó8ò &ò
ò
ð& ñó ðð ñ#ó ð#ð
 ñ"ó ð"ð ò&ó ð&ð òIó ðIð, ò6ó ð6ð ò1ó ð1ð ñ!ó ð!ð ñ1ó ð1õF÷1ð 1r   r   rˆ   )ÚreadÚseekÚtellÚwriteÚreadableÚwritableÚcloser…   c                   ó0   — e Zd Zdd„Zd„ Zed„ «       Zd„ Zy)r‚   Nc                 óf   — || _         || _        || _        || _        |x| _        | _        || _        y r*   )r4   rƒ   r$   rˆ   Ú	blocksizer„   r   )r'   r$   rˆ   r4   rƒ   r„   r   s          r   r&   zArrowFile.__init__æ   s3   € ØˆŒ	ØˆŒ	àˆŒØˆŒà+5Ð5ˆŒ˜œØˆ�r   c                 ó   — | S r*   r"   r,   s    r   Ú	__enter__zArrowFile.__enter__ð   s   € Øˆr   c                 ól   — | j                   j                  «       r| j                   j                  «       S y r*   )rˆ   r…   rO   r,   s    r   rO   zArrowFile.sizeó   s(   € à�;‰;×ÑÔ!Ø—;‘;×#Ñ#Ó%Ð%Ør   c                 ó"   — | j                  «       S r*   )r­   )r'   r   s     r   Ú__exit__zArrowFile.__exit__ù   s   € Ø�z‰z‹|Ðr   r*   )rž   rŸ   r    r&   r²   r£   rO   rµ   r"   r   r   r‚   r‚   Ø   s%   „ óòð ñó ðó
r   r‚   c                   óD   ‡ — e Zd ZdZdZ	 	 	 	 	 	 dˆ fd„	Zed„ «       Zˆ xZS )ÚHadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem
    to connect it's interface with fsspecÚhdfsc                 óP   •— ddl m}  |||||||¬«      }	t        ‰
| �  dd|	i|¤Ž y)a;  

        Parameters
        ----------
        host: str
            Hostname, IP or "default" to try to read from Hadoop config
        port: int
            Port to connect on, or default from Hadoop config if 0
        user: str or None
            If given, connect as this username
        kerb_ticket: str or None
            If given, use this ticket for authentication
        replication: int
            set replication factor of file for write operations. default value is 3.
        extra_conf: None or dict
            Passed on to HadoopFileSystem
        r   )r·   )r0   r1   ÚuserÚkerb_ticketÚreplicationÚ
extra_confr$   Nr"   )r=   r·   r%   r&   )r'   r0   r1   rº   r»   r¼   r½   r   r·   r$   r(   s             €r   r&   zHadoopFileSystem.__init__  s;   ø€ õ6 	0áØØØØ#Ø#Ø!ô
ˆô 	‰ÑÑ)˜BÐ) &Ó)r   c                 óH  — t        | «      }i }|j                  dd «      r|d   |d<   |j                  dd «      r|d   |d<   |j                  dd «      r|d   |d<   |j                  dd «      r4t        |d   «      }|j                  dd «      rt        |d   d   «      |d<   |S )Nr0   Úusernamerº   r1   Ú	url_queryr¼   r   )r	   Úgetr   r€   )r4   r8   ÚoutÚqueriess       r   Ú_get_kwargs_from_urlsz&HadoopFileSystem._get_kwargs_from_urls*  s«   € ä# DÓ)ˆØˆØ�7‰7�6˜4Ô Ø˜f™+ˆC�‰KØ�7‰7�:˜tÔ$Ø˜j™/ˆC�‰KØ�7‰7�6˜4Ô Ø˜f™+ˆC�‰KØ�7‰7�; Ô%Ü˜s ;Ñ/Ó0ˆGØ�{‰{˜=¨$Ô/Ü%(¨°Ñ)?ÀÑ)BÓ%C��MÑ"Øˆ
r   )Údefaultr   NNé   N)	rž   rŸ   r    r¡   r-   r&   ÚstaticmethodrÄ   r¥   r¦   s   @r   r·   r·   ý   s=   ø„ ñ-ð €Hð ØØØØØõ%*ðN ñó ôr   r·   )r   ÚiorU   r^   ra   Ú
contextlibr   Ú	functoolsr   r   Úurllib.parser   Úfsspec.specr   Úfsspec.utilsr   r	   r
   r   r   r#   r   ÚIOBaser‚   r·   r"   r   r   Ú<module>rÏ      s~   ðÛ Û 	Û 	Û Û Ý ß ,Ý !å *÷ó òð$ €ôm1Ð'ô m1ñ` Øò	óô�—	‘	ó óðô0;�~õ ;r   