irdZddlZddlmZmZddlZddlmZddlm Z m Z m Z ddl m Z mZddlmZmZmZmZmZmZmZmZmZmZmZdd lmZmZeeeZee Z ejBd e Dcic]}||c}e" Z#ee"dzejHd d jKedeefZ&edZ'e'jQdgdddddddejRdfdedededee#dzejHdfdede&dededdfd Z*e'jQd!d"d#gddejRdfd$ee"ejVd%fd&ede&dededdf d'Z,e'jQd(gd)ddejRdfd$ee"ejVd%fd*ee"dzejHd+d,fd-ee"dzejHd.fdededdf d/Z-e'jQd0d1d2gejRdfd0ee"ejVd3fdededdfd4Z.ycc}w)5aContains commands to interact with datasets on the Hugging Face Hub. Usage: # list datasets on the Hub hf datasets ls # list datasets with a search query hf datasets ls --search "code" # get info about a dataset hf datasets info HuggingFaceFW/fineweb N) Annotatedget_args)execute_raw_sql_query)CLIErrorRepositoryNotFoundErrorRevisionNotFoundError) DatasetSort_TExpandDatasetProperty_T) AuthorOpt FilterOptFormatWithAutoOptLimitOpt RevisionOpt SearchOptTokenOptapi_object_to_dict get_hf_apimake_expand_properties_parser typer_factory)OutputFormatWithAutooutDatasetSortEnum)typezComma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: z, .)helpcallbackz"Interact with datasets on the Hub.)rz list | ls)zhf datasets lsz*hf datasets ls --sort downloads --limit 10zhf datasets ls --search "code")examples searchauthorfiltersortz Sort results.limitexpandformattokenreturnct|}|r |jnd} |j|||| ||D cgc] } t| } } t j | ycc} w)zList datasets on the Hub.r'N)r"r!r r#r$r%)rvalue list_datasetsrrtable) r r!r"r#r$r%r&r'apisort_key dataset_inforesultss q/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/huggingface_hub/cli/datasets.py datasets_lsr3Es{, 5 !C!tzztH -- .   L <(  IIg sA infoz&hf datasets info HuggingFaceFW/finewebz9hf datasets info my-dataset --expand downloads,likes,tags dataset_idz+The dataset ID (e.g. `username/repo-name`).revisionct|} |j|||}t j |y#t$r}td|d|d}~wt$r}td|d|d|d}~wwxYw) z$Get info about a dataset on the Hub.r*)repo_idr6r%z Dataset 'z ' not found.Nz Revision 'z' not found on 'z'.)rr0rrrrdict)r5r6r%r&r'r.r4es r2 datasets_infor;ks 5 !CU XfU HHTN #D:,l;urlsizerA)headersid_key)rlist_dataset_parquet_filesr>r@rArBrr-) r5r=r>r&r'r.entriesentryfilteredr1s r2datasets_parquetrIs" 5 !C,,Z,OG#*T7%emu{{e?S7HTjrjraf5<<%++eiiQVQ[Q[\jr IIgA%P UsBB6Bsqlzhf datasets sql "SELECT COUNT(*) AS rows FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet')"zhf datasets sql "SELECT * FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet') LIMIT 5" --format jsonzRaw SQL query to execute.c t||}t j |y#t$r}tt||d}~wwxYw)zAExecute a raw SQL query with DuckDB against dataset parquet URLs.) sql_queryr'N)r ImportErrorrstrrr-)rJr&r'resultr:s r2 datasets_sqlrPsD&&EBIIf &s1vA%&s % AAA)/__doc__enumtypingrrtyperhuggingface_hub._dataset_viewerrhuggingface_hub.errorsrrrhuggingface_hub.hf_apir r _cli_utilsr r rrrrrrrrr_outputrrsorted_EXPAND_PROPERTIES _SORT_OPTIONSEnumrNrOptionjoin ExpandOpt datasets_clicommandautor3Argumentr;rIrP)ss0r2rfs  & A[[I    /H%<=>' $))-m/Lm1m/LSVW $JELLdeienenoAeBdCCDE./AB  "FG   4 9 9    $ /* ,       < 0C! 4 9 9 #~u~~3`aab       $ ptZ^ 4 9 9 Q#~u~~3`aabQ cDj,%,,z@j"kk lQ S4Z3U!VV WQ  Q  Q  QQ"  c j!5 9 9  3,GHH I       _0Ms4 H4