Ë
    ëÿæi<  ã                   ór  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 d dlmZmZ d dlmZ d dlmZ ddlmZmZmZ  ej,                  e«      Zh d	£Zh d
£Ze G d„ d«      «       Zdej8                  ez  deeef   fd„Zdeej8                  z  dee eeez  e!z  f      ddfd„Z"deej8                  z  deej8                  z  ddfd„Z#dejH                  dedeej8                  z  e!z  ddfd„Z%deez  e!z  de!fd„Z&dedefd„Z'dedee   ddfd„Z(dejH                  dejR                  de*fd „Z+y)!é    N)Ú	GeneratorÚIterable)Úcontextmanager)Ú	dataclassÚfield)ÚPath)ÚAnyé   )ÚDDUFCorruptedFileErrorÚDDUFExportErrorÚDDUFInvalidEntryNameError>   ú.txtú.jsonú.modelú.safetensors>   úconfig.jsonúscheduler_config.jsonútokenizer_config.jsonúpreprocessor_config.jsonc                   ó€   — e Zd ZU dZeed<   eed<   eed<    ed¬«      Ze	ed<   e
deed	d	f   fd
„«       Zddedefd„Zy	)Ú	DDUFEntrya¿  Object representing a file entry in a DDUF file.

    See [`read_dduf_file`] for how to read a DDUF file.

    Attributes:
        filename (str):
            The name of the file in the DDUF archive.
        offset (int):
            The offset of the file in the DDUF archive.
        length (int):
            The length of the file in the DDUF archive.
        dduf_path (str):
            The path to the DDUF archive (for internal use).
    ÚfilenameÚlengthÚoffsetF)ÚreprÚ	dduf_pathÚreturnNc              #   óN  K  — | j                   j                  d«      5 }t        j                  |j                  «       dt        j                  ¬«      5 }|| j
                  | j
                  | j                  z    –— ddd«       ddd«       y# 1 sw Y   ŒxY w# 1 sw Y   yxY w­w)a-  Open the file as a memory-mapped file.

        Useful to load safetensors directly from the file.

        Example:
            ```py
            >>> import safetensors.torch
            >>> with entry.as_mmap() as mm:
            ...     tensors = safetensors.torch.load(mm)
            ```
        Úrbr   )r   ÚaccessN)r   ÚopenÚmmapÚfilenoÚACCESS_READr   r   )ÚselfÚfÚmms      úx/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.pyÚas_mmapzDDUFEntry.as_mmap:   su   è ø€ ð �^‰^× Ñ  Ô&¨!Ü—‘˜1Ÿ8™8›:¨a¼×8HÑ8HÕIÈRØ˜Ÿ™ t§{¡{°T·[±[Ñ'@ÐAÒA÷ J÷ 'Ð&ßIÐIú÷ 'Ð&üs4   ‚B%ž5BÁ)BÁ<BÂ	B%ÂB	ÂBÂB"ÂB%Úencodingc                 óð   — | j                   j                  d«      5 }|j                  | j                  «       |j	                  | j
                  «      j                  |¬«      cddd«       S # 1 sw Y   yxY w)zÕRead the file as text.

        Useful for '.txt' and '.json' entries.

        Example:
            ```py
            >>> import json
            >>> index = json.loads(entry.read_text())
            ```
        r   )r*   N)r   r!   Úseekr   Úreadr   Údecode)r%   r*   r&   s      r(   Ú	read_textzDDUFEntry.read_textK   sR   € ð �^‰^× Ñ  Ô&¨!Ø�F‰F�4—;‘;ÔØ—6‘6˜$Ÿ+™+Ó&×-Ñ-°xÐ-Ó@÷ '×&Ò&ús   œAA,Á,A5)zutf-8)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚ__annotations__Úintr   r   r   r   r   Úbytesr)   r/   © ó    r(   r   r   #   se   … ñð ƒMØƒKØƒKá Ô'€IˆtÓ'àðB˜ 5¨$°Ð#4Ñ5ò Bó ðBñ A #ð A°Cô Ar9   r   r   r   c                 óX  — i }t        | «      } t        j                  d| › �«       t        j                  t        | «      d«      5 }|j                  «       D ]ž  }t        j                  d|j                  › �«       |j                  t        j                  k7  rt        d«      ‚	 t        |j                  «       t        ||«      }t        |j                  ||j                   | ¬«      ||j                  <   Œ  	 ddd«       d|vrt        d	«      ‚t#        j$                  |d   j'                  «       «      }t)        ||j+                  «       «       t        j                  d
| › dt-        |«      › d�«       |S # t        $ r}t        d|j                  › �«      |‚d}~ww xY w# 1 sw Y   Œ©xY w)a  
    Read a DDUF file and return a dictionary of entries.

    Only the metadata is read, the data is not loaded in memory.

    Args:
        dduf_path (`str` or `os.PathLike`):
            The path to the DDUF file to read.

    Returns:
        `dict[str, DDUFEntry]`:
            A dictionary of [`DDUFEntry`] indexed by filename.

    Raises:
        - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).

    Example:
        ```python
        >>> import json
        >>> import safetensors.torch
        >>> from huggingface_hub import read_dduf_file

        # Read DDUF metadata
        >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf")

        # Returns a mapping filename <> DDUFEntry
        >>> dduf_entries["model_index.json"]
        DDUFEntry(filename='model_index.json', offset=66, length=587)

        # Load model index as JSON
        >>> json.loads(dduf_entries["model_index.json"].read_text())
        {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ...

        # Load VAE weights using safetensors
        >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm:
        ...     state_dict = safetensors.torch.load(mm)
        ```
    zReading DDUF file ÚrzReading entry z)Data must not be compressed in DDUF file.z!Invalid entry name in DDUF file: N)r   r   r   r   úmodel_index.jsonú7Missing required 'model_index.json' entry in DDUF file.zDone reading DDUF file z. Found z entries)r   ÚloggerÚinfoÚzipfileÚZipFiler4   ÚinfolistÚdebugr   Úcompress_typeÚ
ZIP_STOREDr   Ú_validate_dduf_entry_namer   Ú_get_data_offsetr   Ú	file_sizeÚjsonÚloadsr/   Ú_validate_dduf_structureÚkeysÚlen)r   ÚentriesÚzfr?   Úer   Úindexs          r(   Úread_dduf_filerR   [   so  € ðN €GÜ�Y“€IÜ
‡K�KÐ$ Y KÐ0Ô1Ü	�‰œ˜Y›¨Ô	-°Ø—K‘K–MˆDÜ�L‰L˜>¨$¯-©-¨Ð9Ô:Ø×!Ñ!¤W×%7Ñ%7Ò7Ü,Ð-XÓYÐYðiÜ)¨$¯-©-Ô8ô & b¨$Ó/ˆFä%.ØŸ™¨v¸d¿n¹nÐXaô&ˆG�D—M‘MÒ"ñ "÷ 
.ð$  Ñ(Ü$Ð%^Ó_Ð_Ü�J‰J�wÐ1Ñ2×<Ñ<Ó>Ó?€EÜ˜U G§L¡L£NÔ3ä
‡K�KÐ)¨)¨°H¼SÀ»\¸NÈ(ÐSÔTØ€Nøô! -ò iÜ,Ð/PÐQU×Q^ÑQ^ÐP_Ð-`ÓaÐghÐhûðiú÷ 
.Ð	-ús1   ÁAF Â$E6Â9>F Å6	FÅ?FÆFÆF Æ F)rN   c                 ó$  — t         j                  d| › d�«       t        «       }d}t        j                  t        | «      dt        j                  «      5 }|D ]Œ  \  }}||v rt        d|› �«      ‚|j                  |«       |dk(  r-	 t        j                  t        |«      j                  «       «      }	 t        |«      }t         j!                  d	|› d
�«       t#        |||«       ŒŽ 	 ddd«       |€t        d«      ‚	 t%        ||«       t         j                  d| › �«       y# t        j                  $ r}t        d«      |‚d}~ww xY w# t        $ r}t        d|› �«      |‚d}~ww xY w# 1 sw Y   ŒƒxY w# t&        $ r}t        d«      |‚d}~ww xY w)a®  Write a DDUF file from an iterable of entries.

    This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data.
    In particular, you don't need to save the data on disk before exporting it in the DDUF file.

    Args:
        dduf_path (`str` or `os.PathLike`):
            The path to the DDUF file to write.
        entries (`Iterable[tuple[str, Union[str, Path, bytes]]]`):
            An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content.
            The filename should be the path to the file in the DDUF archive.
            The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes.

    Raises:
        - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.).

    Example:
        ```python
        # Export specific files from the local disk.
        >>> from huggingface_hub import export_entries_as_dduf
        >>> export_entries_as_dduf(
        ...     dduf_path="stable-diffusion-v1-4-FP16.dduf",
        ...     entries=[ # List entries to add to the DDUF file (here, only FP16 weights)
        ...         ("model_index.json", "path/to/model_index.json"),
        ...         ("vae/config.json", "path/to/vae/config.json"),
        ...         ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"),
        ...         ("text_encoder/config.json", "path/to/text_encoder/config.json"),
        ...         ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"),
        ...         # ... add more entries here
        ...     ]
        ... )
        ```

        ```python
        # Export state_dicts one by one from a loaded pipeline
        >>> from diffusers import DiffusionPipeline
        >>> from typing import Generator, Tuple
        >>> import safetensors.torch
        >>> from huggingface_hub import export_entries_as_dduf
        >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
        ... # ... do some work with the pipeline

        >>> def as_entries(pipe: DiffusionPipeline) -> Generator[tuple[str, bytes], None, None]:
        ...     # Build a generator that yields the entries to add to the DDUF file.
        ...     # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file.
        ...     # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time)
        ...     yield "vae/config.json", pipe.vae.to_json_string().encode()
        ...     yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict())
        ...     yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode()
        ...     yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict())
        ...     # ... add more entries here

        >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe))
        ```
    zExporting DDUF file 'Ú'NÚwzCan't add duplicate entry: r<   z#Failed to parse 'model_index.json'.zInvalid entry name: zAdding entry 'z' to DDUF filer=   zInvalid DDUF file structure.zDone writing DDUF file )r>   r?   Úsetr@   rA   r4   rE   r   ÚaddrI   rJ   Ú_load_contentr.   ÚJSONDecodeErrorrF   r   rC   Ú_dump_content_in_archiverK   r   )r   rN   Ú	filenamesrQ   Úarchiver   ÚcontentrP   s           r(   Úexport_entries_as_ddufr^       s’  € ôp ‡K�KÐ'¨	 {°!Ð4Ô5Ü“€IØ€EÜ	�‰œ˜Y›¨¬g×.@Ñ.@Ô	AÀWÛ!(ÑˆH�gØ˜9Ñ$Ü%Ð(CÀHÀ:Ð&NÓOÐOØ�M‰M˜(Ô#àÐ-Ò-ðXÜ ŸJ™J¤}°WÓ'=×'DÑ'DÓ'FÓG�EðPÜ4°XÓ>�ô �L‰L˜>¨(¨°>ÐBÔCÜ$ W¨h¸Õ@ñ! ")÷ 
Bð( €}ÜÐWÓXÐXðEÜ  ¨	Ô2ô ‡K�KÐ)¨)¨Ð5Õ6øô% ×+Ñ+ò XÜ)Ð*OÓPÐVWÐWûðXûô
 -ò PÜ%Ð(<¸X¸JÐ&GÓHÈaÐOûðPú÷ 
BÐ	Aûô0 "ò EÜÐ<Ó=À1ÐDûðEúsf   Á1E)Â,D"Â3E	Â>(E)Ã=E5 Ä"EÄ5EÅEÅE)Å		E&ÅE!Å!E&Å&E)Å)E2Å5	FÅ>F
Æ
FÚfolder_pathc                 ó~   ‡— t        ‰«      Šdt        t        t        t         f      fˆfd„}t	        |  |«       «       y)a  
    Export a folder as a DDUF file.

    AUses [`export_entries_as_dduf`] under the hood.

    Args:
        dduf_path (`str` or `os.PathLike`):
            The path to the DDUF file to write.
        folder_path (`str` or `os.PathLike`):
            The path to the folder containing the diffusion model.

    Example:
        ```python
        >>> from huggingface_hub import export_folder_as_dduf
        >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev")
        ```
    r   c               3   ót  •K  — t        ‰«      j                  d«      D ]–  } | j                  «       sŒ| j                  t        vrt
        j                  d| › d�«       Œ@| j                  ‰«      }t        |j                  «      dk\  rt
        j                  d| › d�«       Œƒ|j                  «       | f–— Œ˜ y ­w)Nz**/*zSkipping file 'z' (file type not allowed)é   z"' (nested directories not allowed))r   ÚglobÚis_fileÚsuffixÚDDUF_ALLOWED_ENTRIESr>   rC   Úrelative_torM   ÚpartsÚas_posix)ÚpathÚpath_in_archiver_   s     €r(   Ú_iterate_over_folderz3export_folder_as_dduf.<locals>._iterate_over_folder  s£   øè ø€ Ü˜Ó%×*Ñ*¨6Ö2ˆDØ—<‘<”>ØØ�{‰{Ô"6Ñ6Ü—‘˜¨t¨fÐ4MÐNÔOØØ"×.Ñ.¨{Ó;ˆOÜ�?×(Ñ(Ó)¨QÒ.Ü—‘˜¨t¨fÐ4VÐWÔXØØ!×*Ñ*Ó,¨dÐ2Ó2ñ 3ùs   ƒB5B8N)r   r   Útupler4   r^   )r   r_   rl   s    ` r(   Úexport_folder_as_ddufrn   ù   s9   ø€ ô$ �{Ó#€Kð3¤(¬5´´d°Ñ+;Ñ"<õ 3ô ˜9Ñ&:Ó&<Õ=r9   r\   r   r]   c                 óv  — | j                  |dd¬«      5 }t        |t        t        f«      r=t        |«      }|j                  d«      5 }t	        j
                  ||d«       d d d «       n1t        |t        «      r|j                  |«       nt        d|› d�«      ‚d d d «       y # 1 sw Y   ŒxY w# 1 sw Y   y xY w)NrU   T)Úforce_zip64r   i  € zInvalid content type for z. Must be str, Path or bytes.)	r!   Ú
isinstancer4   r   ÚshutilÚcopyfileobjr7   Úwriter   )r\   r   r]   Ú
archive_fhÚcontent_pathÚ
content_fhs         r(   rZ   rZ     sŸ   € Ø	�‰�h °ˆÔ	6¸*Ü�g¤¤T˜{Ô+Ü ›=ˆLØ×"Ñ" 4Ô(¨JÜ×"Ñ" :¨z¸?ÔK÷ )Ð(ä˜¤Ô'Ø×Ñ˜WÕ%ä!Ð$=¸h¸ZÐGdÐ"eÓfÐf÷ 
7Ð	6÷ )Ð(ú÷ 
7Ð	6ús#   •3B/ÁB#Á :B/Â#B,	Â(B/Â/B8c                 ó´   — t        | t        t        f«      rt        | «      j                  «       S t        | t        «      r| S t        dt        | «      › d�«      ‚)zoLoad the content of an entry as bytes.

    Used only for small checks (not to dump content into archive).
    z6Invalid content type. Must be str, Path or bytes. Got Ú.)rq   r4   r   Ú
read_bytesr7   r   Útype)r]   s    r(   rX   rX   )  sR   € ô
 �'œC¤˜;Ô'Ü�G‹}×'Ñ'Ó)Ð)Ü	�GœUÔ	#ØˆäÐ VÔW[Ð\cÓWdÐVeÐefÐgÓhÐhr9   Ú
entry_namec                 óê   — d| j                  d«      d   z   t        vrt        d| › �«      ‚d| v rt        d| › d�«      ‚| j                  d«      } | j	                  d«      dkD  rt        d| › d�«      ‚| S )	Nry   éÿÿÿÿzFile type not allowed: Ú\z0Entry names must use UNIX separators ('/'). Got Ú/é   z-DDUF only supports 1 level of directory. Got )Úsplitrf   r   ÚstripÚcount)r|   s    r(   rF   rF   6  s”   € Ø
ˆZ×Ñ˜cÓ" 2Ñ&Ñ&Ô.BÑBÜ'Ð*AÀ*ÀÐ(NÓOÐOØˆzÑÜ'Ð*ZÐ[eÐZfÐfgÐ(hÓiÐiØ×!Ñ! #Ó&€JØ×Ñ˜Ó˜qÒ Ü'Ð*WÐXbÐWcÐcdÐ(eÓfÐfØÐr9   rQ   Úentry_namesc                 ó@  ‡‡— t        | t        «      st        dt        | «      › d�«      ‚‰D �ch c]  }d|v sŒ|j	                  d«      d   ’Œ }}|D ]D  Š‰| vrt        d‰› d�«      ‚t        ˆˆfd„t        D «       «      rŒ0t        d‰› d	t        › d�«      ‚ y
c c}w )a�  
    Consistency checks on the DDUF file structure.

    Rules:
    - The 'model_index.json' entry is required and must contain a dictionary.
    - Each folder name must correspond to an entry in 'model_index.json'.
    - Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json').

    Args:
        index (Any):
            The content of the 'model_index.json' entry.
        entry_names (Iterable[str]):
            The list of entry names in the DDUF file.

    Raises:
        - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).
    z>Invalid 'model_index.json' content. Must be a dictionary. Got ry   r€   r   zMissing required entry 'z' in 'model_index.json'.c              3   ó0   •K  — | ]  }‰› d |› �‰v –— Œ y­w)r€   Nr8   )Ú.0Úrequired_entryr…   Úfolders     €€r(   Ú	<genexpr>z+_validate_dduf_structure.<locals>.<genexpr>Z  s%   øè ø€ ÐrÑUqÀ>�f�X˜Q˜~Ð.Ð/°;Ô>ÑUqùs   ƒz!Missing required file in folder 'z!'. Must contains at least one of N)rq   Údictr   r{   r‚   ÚanyÚDDUF_FOLDER_REQUIRED_ENTRIES)rQ   r…   ÚentryÚdduf_foldersrŠ   s    `  @r(   rK   rK   A  sµ   ù€ ô$ �eœTÔ"Ü$Ð'eÔfjÐkpÓfqÐerÐrsÐ%tÓuÐuá5@ÓQ±[¨EÀCÈ5ÂL�E—K‘K Ó$ QÓ'°[€LÐQÛˆØ˜ÑÜ(Ð+CÀFÀ8ÐKcÐ)dÓeÐeÜÔrÕUqÓrÕrÜ(Ø3°F°8Ð;\Ô]yÐ\zÐz{Ð|óð ñ	 ùò Rs
   ¯	B¹BrO   r?   c                 ód  — | j                   €t        d«      ‚|j                  }| j                   j                  |«       | j                   j	                  d«      }t        |«      dk  rt        d«      ‚t        j                  |dd d«      }t        j                  |dd d«      }|dz   |z   |z   }|S )a1  
    Calculate the data offset for a file in a ZIP archive.

    Args:
        zf (`zipfile.ZipFile`):
            The opened ZIP file. Must be opened in read mode.
        info (`zipfile.ZipInfo`):
            The file info.

    Returns:
        int: The offset of the file data in the ZIP archive.
    z+ZipFile object must be opened in read mode.é   zIncomplete local file header.é   é   Úlittle)Úfpr   Úheader_offsetr,   r-   rM   r6   Ú
from_bytes)rO   r?   r—   Úlocal_file_headerÚfilename_lenÚextra_field_lenÚdata_offsets          r(   rG   rG   `  s²   € ð 
‡u�u€}Ü$Ð%RÓSÐSð ×&Ñ&€Mð ‡E�E‡J�Jˆ}ÔØŸ™Ÿ
™
 2›Ðä
ÐÓ Ò"Ü$Ð%DÓEÐEô —>‘>Ð"3°B°rÐ":¸HÓE€LÜ—n‘nÐ%6°r¸"Ð%=¸xÓH€Oð   "Ñ$ |Ñ3°oÑE€KàÐr9   ),rI   Úloggingr"   Úosrr   r@   Úcollections.abcr   r   Ú
contextlibr   Údataclassesr   r   Úpathlibr   Útypingr	   Úerrorsr   r   r   Ú	getLoggerr0   r>   rf   rŽ   r   ÚPathLiker4   rŒ   rR   rm   r7   r^   rn   rA   rZ   rX   rF   rK   ÚZipInfor6   rG   r8   r9   r(   Ú<module>r¨      s¹  ðÛ Û Û Û 	Û Û ß /Ý %ß (Ý Ý ç WÑ Wð 
ˆ×	Ñ	˜8Ó	$€òÐ ò Ð ð ÷4Að 4Aó ð4AðnB˜bŸk™k¨CÑ/ð B°D¸¸i¸Ñ4Hó BðJV7 c¨B¯K©KÑ&7ð V7À(È5ÐQTÐVYÐ\`ÑV`ÐchÑVhÐQhÑKiÑBjð V7Ðosó V7ðr!> S¨2¯;©;Ñ%6ð !>ÀSÈ2Ï;É;ÑEVð !>Ð[_ó !>ðH	g g§o¡oð 	gÀð 	gÈsÐUW×U`ÑU`ÑO`ÐchÑOhð 	gÐmqó 	gð
i˜3 ™:¨Ñ-ð 
i°%ó 
ið¨#ð °#ó ð Cð °h¸s±mð Èó ð>"˜Ÿ™ð "°·±ð "ÀCô "r9   