(tiu& dZddlZddlZddlZddlmZddlmZddlZddlm Z ddl m Z m Z m Z mZmZmZmZddlmZmZmZmZmZdd lmZmZmZmZmZmZmZm Z dd l!m"Z"e jFe$Z%e d Z&d e'de(fdZ)de'de*e'e'ffdZ+dde,e-zde(de'fdZ.ddedzde(de'fdZ/ dde0e e zde(de(de0e'fdZ1 dde2de0e'de'de,de,de(ddfd Z3e&jid!gd"# dd$ee'ejjd% fd&ee(ejld'd( fd)ee(ejld*d+ fded,eddf d-Z7de'de(fd.Z8e&jid/gd0#ddddejrddfdee'dzejjd1 fdee(ejld2d3d4 fd5ee(ejld6d7 fd8ee(ejld9d:d; fde'dzde(d5e(d8e(de&jidPgdQ# ddee'ejjdR fd8ee(ejld9d:dS fdHee(ejldIdJdK fdTee(ejldUdV fdWee0e'dzejldX fdYee0e'dzejldZ fded,eddfd[Z?e&jid\gd]# dd^ee'ejjd_ fd`ee'ejjda fd,eddfdbZ@e&jidcgdd# ddeee'dzejjdf fdgee'dzejjdh fdFee(ejldi fdjee(ejldkdl fdmee(ejldndo fdpee'dzejldq fdree'dzejlds fdTee(ejldUdt fdWee0e'dzejldu fdYee0e'dzejldv fdwee'dzejldx fdyee(ejldzd{ fd|ee(ejld}d~ fdee(ejlddd fdee(ejlddd fd,eddf"dZAe&jidgd# ddee'ejjd fdee'dzejjd fded,eddf dZBy)z7Contains commands to interact with buckets via the CLI.N)datetime) Annotated)logging) BUCKET_PREFIX BucketFile BucketFolder FilterMatcher_is_bucket_path_parse_bucket_path_split_bucket_id_and_prefix)SoftTemporaryDirectory StatusLineare_progress_bars_disableddisable_progress_barsenable_progress_bars) FormatOpt OutputFormatQuietOptTokenOptapi_object_to_dict get_hf_apiprint_list_output typer_factory)outz"Commands to interact with buckets.)helppathreturnc$|jdS)Nzhf://) startswith)rs j/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/huggingface_hub/cli/buckets.py _is_hf_handler"=s ??7 ##argumentc|jtr t|S t|S#t$rt d|dtdwxYw)zParse a bucket argument accepting both 'namespace/name(/prefix)' and 'hf://buckets/namespace/name(/prefix)'. Returns: tuple: (bucket_id, prefix) where bucket_id is "namespace/bucket_name" and prefix may be empty string. zInvalid bucket argument: -. Must be in format namespace/bucket_name or znamespace/bucket_name)r rr r ValueError)r$s r!_parse_bucket_argumentr(Asb =)!(++ *844  'z2 /!6 8   s - A Fsizehuman_readablecz|s t|SdD]$}|dkr|dk(r |d|cS|dd|cS|dz}&|ddS)zFormat a size in bytes.)BKBMBGBTBir, z.1fz PB)str)r)r*units r! _format_sizer4Rse 4y- $;s{q''3Zq' '   . 3Zs r#mtimecP|y|r|jdS|jdS)z0Format mtime datetime to a readable date string.z %b %d %H:%Mz%Y-%m-%d %H:%M:%S)strftime)r5r*s r! _format_mtimer9`s, }~~m,, >>- ..r#itemsquietc i}|D]e}|jjd}|}|ddD]}||vrdii||<||d}|d}t|tr ||vsWdii||<_d|i||<gd} d} d} |s{|D]i}t|tst |j |} t| t| } t|j|} t| t| } k| dkDr| dz| z} g}t||d| | | |S) aBuild a tree representation of files and directories. Produces ASCII tree with size and date columns before the tree connector. When quiet=True, only the tree structure is shown (no size/date). Args: items: List of BucketFile/BucketFolder items human_readable: Whether to show human-readable sizes and short dates quiet: If True, show only the tree structure without sizes/dates Returns: List of formatted tree lines /N __children____item__rr7 prefix_widthmax_size_widthr*) rsplit isinstancerrr4r)maxlenr9r5 _render_tree)r:r*r;treeitempartscurrentpart final_partrCrDmax_date_widthsize_strdate_strliness r! _build_treerTisB&D $#2JD7"!/ 4 dmN3G 2Y dL )('5r&: ##-t"4GJ  LNN D$ +' >B!$^S]!C(^D!$^S]!C  A )A->LE   !%%  Lr#noderSindentrCrDc ,t|j}t|D]\}\}} |t|dz k(} | rdnd} d| v} | j di} |dkDry| rd|z}nP| j d}|8t |j |}t|j|}|d |d |}nd|z}|j|d || || rd nd n|j|| || rd nd | s|| rd ndz}t| |||||y)z:Recursively render a tree structure with size+date prefix.ru └── u ├── r?rr1r@N> r=r7z u│ rB) sortedr: enumeraterHgetr4r)r9r5appendrI)rUrSrVrCrDr*r:inamevalueis_last connectoris_dirchildrenprefixrKrQrR child_indents r!rIrIsE 4::< E%e,=D%s5zA~%$+L 5(99^R0 ! |+yy,#+DII~FH,TZZHH (>*:(:;2hZHF </F LLF82fXi[fsRT>UV W LLF8I;tfFS4KL M !wVHEL )--  1-r#create)zhf buckets create my-bucketz hf buckets create user/my-bucketz-hf buckets create hf://buckets/user/my-bucketz*hf buckets create user/my-bucket --privatez+hf buckets create user/my-bucket --exist-ok)r_examples bucket_idzTBucket ID: bucket_name, namespace/bucket_name, or hf://buckets/namespace/bucket_nameprivatez --privatezCreate a private bucket.exist_okz --exist-okz3Do not raise an error if the bucket already exists.tokenct|}|jtr3 t|\}}|r t j d|dtd|}|j||r|nd|} |rt| jytd| jd| jd y#t$r#}t j t|d}~wwxYw) zCreate a new bucket.rlNz-Cannot specify a prefix for bucket creation: . Use namespace/bucket_name or namespace/bucket_name.)rjrkzBucket created: z (handle: )) rr rr(r'typer BadParameterr2 create_bucketprinthandleurl) rirjrkr;rlapi parsed_idree bucket_urls r!rgrgsF 5 !CM* - 6y A Iv $$? {K11>?UW  """#J   j    0 :;L;L:MQOP# -$$SV, , -sB,, C5CCcd|jtr|ttd}d|vS|}d|vS)zFCheck if argument is a bucket ID (namespace/name) vs just a namespace.Nr=)r rrH)r$rs r! _is_bucket_idr}s?=)M*,- $; $;r#z list | ls) zhf buckets listzhf buckets list huggingfacezhf buckets list user/my-bucketz!hf buckets list user/my-bucket -Rz!hf buckets list user/my-bucket -hz%hf buckets list user/my-bucket --treez(hf buckets list user/my-bucket --tree -hz+hf buckets list hf://buckets/user/my-bucketz%hf buckets list user/my-bucket/sub -RzyNamespace (user or org) to list buckets, or bucket ID (namespace/bucket_name(/prefix) or hf://buckets/...) to list files.z--human-readablez-hz$Show sizes in human readable format.as_treez--treez3List files in tree format (only for listing files). recursivez --recursivez-Rz0List files recursively (only for listing files).formatc t|duxr t|}|rt|||||||yt|||||||y)zList buckets or files in a bucket. When called with no argument or a namespace, lists buckets. When called with a bucket ID (namespace/bucket_name), lists files in the bucket. N)r$r*r~rrr;rl) namespacer*r~rrr;rl)r} _list_files _list_buckets)r$r*r~rrr;rl is_file_modes r!list_cmdrsXr4'CM(,CL)  ) r#rc*|rtjd|rtjd|8|jtr#|t td}|j d}t |}|j|Dcgc] }t|} }| s<|s:|tjk7r'||n|jd} td| d ygd } d td ttffd } ddd} t!| ||| | | ycc}w)zList buckets in a namespace.z'Cannot use --tree when listing buckets.z,Cannot use --recursive when listing buckets.Nr=rn)rr_z"No buckets found under namespace ''.)idrjr) total_files created_atrKrc ddlm}||jd||jdt|jdd||jd||jd gS) Nr) _format_cellrrjr)rr*rr) _cli_utilsrr\r4)rKrr*s r!row_fnz_list_buckets..row_fnsj, $ ( ), - &!,^ L -0 1 ,/ 0   r#right)r)r)rr;headersr alignments)rrrsr rrHrstripr list_bucketsrrjsonwhoamirudictlistr2r)rr*r~rrr;rlrxbucketresultsresolved_namespacerrrs ` r!rrjs   !JKK  !OPP!5!5m!Dc-023 $$S) 5 !C8;8H8HS\8H8]^8]f!&)8]G^ <#4#44.7.CV\I]  67I6J"M N DG  T  d3i  "':JgfE7SYfpq-_s Dc X|r(|tjk(rtjdt |} t |\}} t|j|| xsd|} | s tdytd| D} |tjk(r:| D cgc] } t| }} ttj|dn|r!t| || }|D] }t|n|rF| D]@} t| t rt| j"d ,t| j"Bn| D]} t| t r7t%| j&|}td d d |dd | j"d Jt)| j*|}t%| j,|}t|d d |dd | j"|s| rt/j1dyyy#t $r#} tjt| d} ~ wwxYwcc} w)zList files in a bucket.z%Cannot use --tree with --format json.rnNrerz(empty)c3<K|]}t|tywN)rFr).0rKs r! z_list_files..sKUT*T<8UsrArV)r*r;r=r7z>12rYz>19z!Use -R to list files recursively.)rrrrrsrr(r'r2rlist_bucket_treeruanyrdumpsrTrFrrr9 uploaded_atr4r)r5rdone)r$r*r~rrr;rlrxrirerzr:has_directoriesrKr tree_linesline mtime_strrQs r!rrs6\...  !HII 5 !C)28< 6   >T   E  iKUKKO """8=>%d+> djj+,  ~US D $K D$ - 1o&dii D$ -)$*:*:NK C9S/DII;a@A' >B)$**nE #b3r$))EF  =>)9W )  Q(()&?sG8H'8 H$HH$infozhf buckets info user/my-bucketz+hf buckets info hf://buckets/user/my-bucketzFBucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_namec<t|} t|\}}|j |}|rt|jyttjt|dy#t$r#}tjt |d}~wwxYw)zGet info about a bucket.rnNrAr) rr(r'rrrsr2 bucket_inforurrrr)rir;rlrxry_rzrs r!rrs$ 5 !C)-i8 1__Y 'F  fii djj+F3A>? )  Q(()sA// B8BBdelete)z hf buckets delete user/my-bucketz-hf buckets delete hf://buckets/user/my-bucketz&hf buckets delete user/my-bucket --yesz-hf buckets delete user/my-bucket --missing-okyesz--yesz-yzSkip confirmation prompt. missing_okz --missing-okz3Do not raise an error if the bucket does not exist.c|jtr4 t|\}}|r t j d|dtd|}n$d|vr t j d|dtdtjd|d | t| }|j|| |r t|ytd |y#t$r#}t j t |d}~wwxYw)zDelete a bucket. This deletes the entire bucket and all its contents. Use `hf buckets rm` to remove individual files. Nz-Cannot specify a prefix for bucket deletion: rorpr=zInvalid bucket ID: r&z(Are you sure you want to delete bucket ''?rrn)rzBucket deleted: ) r rr(r'rrrsr2rconfirmr delete_bucketru) rirrr;rlryrerzrxs r!rrsLM* - 6y A Iv $$? {K11>?UW  I   !)-;;H/I_ a  KK:9+RHcR 5 !CiJ7  i   ,-+ -$$SV, , -sC C1C,,C1z remove | rm)z)hf buckets remove user/my-bucket/file.txtz2hf buckets rm hf://buckets/user/my-bucket/file.txtz.hf buckets rm user/my-bucket/logs/ --recursivez:hf buckets rm user/my-bucket --recursive --include "*.tmp"z8hf buckets rm user/my-bucket/data/ --recursive --dry-runzBucket path: namespace/bucket_name/path or hf://buckets/namespace/bucket_name/path. With --recursive, namespace/bucket_name is also accepted to target all files.z0Remove files recursively under the given prefix.dry_runz --dry-runz8Preview what would be deleted without actually deleting.includezQInclude only files matching pattern (can specify multiple). Requires --recursive.excludezLExclude files matching pattern (can specify multiple). Requires --recursive.c| t|\}} | dk(r|stjd|d|d|s|r|stjdt |} |rt | } | jd g} | j|| jd xsdd D]B}t|ts| j|| jd t| dD| jd t| d|s|r9t||}| Dcgc] }|j!|j"s|"}}n| }|Dcgc]}|j"}}t%d|D}t'|d }|s|s t)dyt|d|}|s7|s5|s|D]}t)d|t+j,d|d|dd|r.|D]}t)dt.|d |t)d|dy| j1|||r|D] }t)|y|D]}t)dt.|d |t)d|d|dy| jd }|stjd |r#t)dt.|d |t)d!yt+j,d"|d#|d|| j1||g|r t)|yt)dt.|d |y#t$r#} tjt | d} ~ wwxYwcc}wcc}w)$zbRemove files from a bucket. To delete an entire bucket, use `hf buckets delete` instead. Nr7z?No file path specified. To remove files, provide a path (e.g. 'zd/FILE') or use --recursive to remove all files. To delete the entire bucket, use `hf buckets delete z`.z,--include and --exclude require --recursive.rn)enabledzListing files from remoter=TrzListing files from remote (z files))include_patternsexclude_patternsc34K|]}|jywr)r))rfs r!rzremove..s7AsrzNo files to remove.z file(s) totaling rYzRemove z from 'rFrzdelete: z (dry run) z would be removed.)rzRemoved rzFile path cannot be empty.z"(dry run) 1 file would be removed.zRemove 'z' from ')r(r'rrrsr2rrupdaterrrFrr]rHrr matchesrsumr4rurrrbatch_bucket_files)r$rrrrrr;rlrirerzrxstatus all_filesrKmatcherr matched_files file_paths total_sizerQ count_labelr file_paths r!remover<sa|)28< 6|I   k"DDM;b R  7I  !OPP 5 !CI. 12&( (( ==%-) D $ +  & ;C N;K7ST   1#i.1AIJ g#WwWG(1M 1W__QVV5LQ MM%M&34maffm 4777  4@+, Z));H:F 7&DBtf+&' KK'+gi[C O " {!D6BC# J{m+=> ?  y< "d ## {!D6BC# H[M 2> ?MM#& $$%AB B  H]OI;a {C D 6 7  hyk)B?SI y)=  )  H]OI;a {C Di )  Q(()@N5s)L L4=L4 L9 L1L,,L1move)z/hf buckets move user/old-bucket user/new-bucketz/hf buckets move user/my-bucket my-org/my-bucketzIhf buckets move hf://buckets/user/old-bucket hf://buckets/user/new-bucketfrom_idzMSource bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_nameto_idzRDestination bucket ID: namespace/bucket_name or hf://buckets/namespace/bucket_namec$t|\}}|r tjd|dtdt|\}}|r tjd|dtdt |}|j ||t d|d|y) z2Move (rename) a bucket to a new name or namespace.z)Cannot specify a prefix for bucket move: rorprn)rrzBucket moved:  -> N)r(rrrsrr move_bucketru)rrrlparsed_from_id from_prefix parsed_to_id to_prefixrxs r!rrs2#9"ANK  7yA--:O;Q S  5U;L)  7w?--:O;Q S  5 !COON,O? N>*$|n =>r#sync)z2hf buckets sync ./data hf://buckets/user/my-bucketz2hf buckets sync hf://buckets/user/my-bucket ./dataz;hf buckets sync ./data hf://buckets/user/my-bucket --deletez^hf buckets sync hf://buckets/user/my-bucket ./data --include "*.safetensors" --exclude "*.tmp"zIhf buckets sync ./data hf://buckets/user/my-bucket --plan sync-plan.jsonlz'hf buckets sync --apply sync-plan.jsonlzhf buckets cp my-config.json hf://buckets/user/my-bucket/logs/zKhf buckets cp my-config.json hf://buckets/user/my-bucket/remote-config.jsonz7hf buckets cp - hf://buckets/user/my-bucket/config.jsonzVhf buckets cp hf://buckets/user/my-bucket/logs/ hf://buckets/user/archive-bucket/logs/zehf buckets cp hf://datasets/user/my-dataset/processed/ hf://buckets/user/my-bucket/dataset/processed/srczMSource: local file, any hf:// handle (model, dataset, bucket), or - for stdindstz@Destination: local path, bucket hf://... handle, or - for stdoutc  t|}t|}|duxr t|}t|}|duxr t|}|dk(} |dk(} |rC|rA|r t |j || |r t |std|d|y|s0|s.| s,|t jdt jd| r|st jd| r;|r9t|\} } | d k(s| jd rt jd | r|st jd |s6| s4tjj|rt jd |rt|\}} | d k(s| jd rt jd| j!d dd}| rt#}|s t t%5}tjj'||}|j)|| |fgt+|d5}|j-dx}r=t.j0j2j5||j-dx}r=dddddd|sty||}nrtjj|s0|jtj6s|jd r!tjj'||}n|}tjj9|}|rtj:|d|r t |j)|| |fg|r t |std|d|yyy| rxt|\}}t.j<j2j-}|r t |j?|||fg|r t |std|yytjjA|st jd|t|\}} | d k(r tjjC|}n6| jd r#| tjjC|z}n| }|r t |j?|||fg|r t |std|dtD|d |yy#t $r#} t jt| d} ~ wwxYw#|r twwxYw#1swYxYw#1swYxYw#|s twwxYw#|r twwxYw#|r twwxYw#|r twwxYw)zCopy files to or from buckets.rnN-zCopied: rz2Missing destination. Provide a bucket path as DST.z;One of SRC or DST must be a bucket path (hf://buckets/...).z+Stdin upload requires a bucket destination.r7r=zAStdin upload requires a full destination path including filename.z"Cannot pipe to stdout for uploads.zNSource must be a file, not a directory. Use `hf buckets sync` for directories.zJSource path must include a file name, not just a bucket or directory path.rr>rbiHT)rkz Downloaded: )addzUploaded: stdin -> zSource file not found: z Uploaded: )#rr"r r copy_filesr'rrrsr2rrur endswithosrisdirrsplitrr joindownload_bucket_filesopenreadsysstdoutbufferwritesepdirnamemakedirsstdinrisfilebasenamer)rrr;rlrx src_is_hf dst_is_hf src_is_bucket dst_is_bucket src_is_stdin dst_is_stdoutrzrrerifilenamepbar_was_disabledtmp_dirtmp_pathrchunk local_path parent_dir remote_pathdatas r!rrs4 5 !Cc"I46M#$6I#C(MtO<(H--i68:L9MNh-'(vvj'99e9JJ--33E:()vvj'99e9..)(*{% s#s||BFF';s||C?PWW\\#x8  4J J6%' +)))vz6J5KL(* SEj\:;1)6 !3C!8 ;yy$$&  ! # '  " "9D+3F2G " H$& 'u- . ww~~c"$$'>se%DE E.s3 6 R<''**3/K __S ! 277#3#3C#88K K  ! # '  " "9C3E2F " G$& Jse4 yk;-P Qm -$$SV, , -$&\.-.-)(*)*(* $&2$&s S- U!AT;#AT.4T;<U U U,,U>- T6TTTT+.T8 3T;;UUUU),U;>V )F)FF)rrF)FFFN)FN)FFFNNFNr)NNFFFNNFNNNFFFFN)NFN)C__doc__rrrrtypingrrrhuggingface_hubrhuggingface_hub._bucketsrrrr r r r huggingface_hub.utilsr rrrrrrrrrrrrr_outputr get_logger__name__logger buckets_clir2boolr"tupler(intfloatr4r9rrTrrIcommandArgumentOptionrgr}tablerrrrrrrrrr#r!r"s >  #      H %!EF $$$ S U38_ " sU{ D S /D/$/3/!<  \) *<< < #Y <F ) ) 9) ) )  )  ) )X   .  -/Q g  /Q  +  /Q  F  /Q* +/Q, -/Q. //Q /QdCD    .    $**G@  d W  @    7  @ $  F  %@ 2   C  3@ B C@ D E@ F G@ H I@  @ F-rTz-r-r-r -r  -r  -r :-r -r`>?>?>?>? >?  >?  >? :>? >?B  (5@ Y  @ @ @ @@2  .  /5. Y  5.    ,   5.  F  5., -5.. /5.0 15.5.p   6    ]JE a  JE   C  JE$    ,   %JE4  K  5JEB S D d  CJEN S D _  OJEZ [JE\ ]JE^ _JE JEZ  *"?  `  "?  e   "? "? "?"?T    &          Ix  d ^   x  d c   x   B   x&  N  'x4  O  5xB  d  P   CxN  d  6   OxZ  H  [xh S D I  ixt S D I  ux@ d  ;  AxL  T  MxZ  V  [xh   8  ixx    "   yxH IxJ Kx x@   * QR  ^U^^!p qq QR  d NENN(jkk  QR QR QR QRQRr#