o 0i'@sdZddlZddlZddlmZmZmZddlZddlm Z ddl m Z m Z m Z ddlmZmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZe eeZ!eeZ"ej#d d d e"De$d Z%eee$ej&d d'e!dee!dfZ(eddZ)e)j*dgddddddddej+ddf dedededeee%ej&ddfdede(ded ed!ed"dfd#d$Z,e)j*d%d&d'gd   d9sr)typezComma-separated properties to return. When used, only the listed properties (and id) are returned. Example: '--expand=downloads,likes,tags'. Valid: z, .)helpcallbackz"Interact with datasets on the Hub.)r"z list | ls)zhf datasets lsz*hf datasets ls --sort downloads --limit 10zhf datasets ls --search "code")Zexamples Fsearchauthorfiltersortz Sort results.limitexpandformatquiettokenreturnc CsJt|d} |r |jnd} dd| j|||| ||dD} t| ||ddS)zList datasets on the Hub.r-NcSsg|]}t|qSr)r)r dataset_inforrr aszdatasets_ls..)r'r&r%r(r)r*r+r,)rvalueZ list_datasetsr) r%r&r'r(r)r*r+r,r-apisort_keyresultsrrr datasets_lsHs  r7infoz&hf datasets info HuggingFaceFW/finewebz9hf datasets info my-dataset --expand downloads,likes,tags dataset_idz+The dataset ID (e.g. `username/repo-name`).revisionc Cst|d}z |j|||d}Wn,ty$}z td|d|d}~wty;}z td|d|d|d}~wwttjt|d d dS) z>Get info about a dataset on the Hub. Output is in JSON format.r/)repo_idr:r*z Dataset 'z ' not found.Nz Revision 'z' not found on 'z'.)indent) rr0rrrprintjsondumpsr)r9r:r*r-r4r8errr datasets_infoos rBZparquet)z(hf datasets parquet cfahlgren1/hub-statsz8hf datasets parquet cfahlgren1/hub-stats --subset modelsz6hf datasets parquet cfahlgren1/hub-stats --split trainz6hf datasets parquet cfahlgren1/hub-stats --format jsonsubsetz--subsetz(Filter parquet entries by subset/config.splitz Filter parquet entries by split.c sLt|d}|j||d}fdd|D}dd|D} t| ||dddS) z/List parquet file URLs available for a dataset.r/)r;configcs"g|] }dus |jkr|qS)NrDrentryrFrrr1s"z$datasets_parquet..cSs"g|] }|j|j|j|jdqS))rCrDurlsize)rErDrIrJrGrrrr1srI)r+r,Zid_keyN)rZlist_dataset_parquet_filesr) r9rCrDr+r,r-r4entriesfilteredr6rrFrdatasets_parquets rMsqlzhf datasets sql "SELECT COUNT(*) AS rows FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet')"zhf datasets sql "SELECT * FROM read_parquet('https://huggingface.co/api/datasets/cfahlgren1/hub-stats/parquet/models/train/0.parquet') LIMIT 5" --format jsonzRaw SQL query to execute.c CsJzt||d}Wnty}ztt||d}~wwt||dddS)zAExecute a raw SQL query with DuckDB against dataset parquet URLs.)Z sql_queryr-NFr2)r ImportErrorrstrr)rNr+r-resultrArrr datasets_sqls rR)NNN)1__doc__enumr?typingrrrZtyperZhuggingface_hub._dataset_viewerrZhuggingface_hub.errorsrrrZhuggingface_hub.hf_apir r Z _cli_utilsr r rrrrrrrrrrrrsortedZ_EXPAND_PROPERTIESZ _SORT_OPTIONSEnumrPrOptionjoinZ ExpandOptZ datasets_clicommandtabler7ZArgumentrBrMrRrrrrs   @