+ NV-j_Rt^RIHt^RIt^RIt^RIt^RIHtHtH t ^RI H t H t ^RI Ht^RIHt^RIHtHt^RIHt^R IHt^R IHtHt^R IHt^R IHtHtH t H!t!H"t"]'d^R I#H$t$H%t%H&t&H't'H(t(H)t)RRlt*RRRllt+!RR4t,!RR],4t-!RR],4t.RRRllt/]!R4RRR]P`RRR3RRll4t1R#) zparquet compat) annotationsN) TYPE_CHECKINGAnyLiteral)catch_warningsfilterwarnings)lib)import_optional_dependency)AbstractMethodErrorPandas4Warning) set_module)check_dtype_backend) DataFrame get_option)arrow_table_to_pandas) IOHandles get_handle is_fsspec_urlis_urlstringify_path) DtypeBackendFilePathParquetCompressionOptions ReadBufferStorageOptions WriteBufferc V^8dQhRRRR/#)enginestrreturnBaseImpl)formats"b/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/pandas/io/parquet.py __annotate__r%4sGGsGxGcLVR8Xd \R4pVR8Xd.\\.pRpVF pV!4u# \RV 24hVR8Xd \4#VR8Xd \4#\ R 4h \d%pTR\ T4,, pRp?KtRp?ii;i) zreturn our implementationautozio.parquet.enginez - NzUnable to find a usable engine; tried using: 'pyarrow', 'fastparquet'. A suitable version of pyarrow or fastparquet is required for parquet support. Trying to import the above resulted in these errors:pyarrow fastparquetz.engine must be one of 'pyarrow', 'fastparquet')r PyArrowImplFastParquetImpl ImportErrorr ValueError)rengine_classes error_msgs engine_classerrs& r$ get_enginer44s /0 %7 *L 1#~%+  C l    } =   E FF% 1gC00  1sA44 B#?BB#c 0V^8dQhRRRRRRRRR R R R /#) rpathz1FilePath | ReadBuffer[bytes] | WriteBuffer[bytes]fsrstorage_optionsStorageOptions | Nonemoderis_dirboolr zVtuple[FilePath | ReadBuffer[bytes] | WriteBuffer[bytes], IOHandles[bytes] | None, Any]r")r#s"r$r%r%VsD<'<' ;<' <'+<'  <'  <'  <'r&c\V4pVe\RRR7p\RRR7pVe0\WP4'dV'd \ R4hMKVe'\WP P 4'dM!\R\V4P 24h\V4'dvVfrVf5\R4p\R4pVPPV4wrVf5\R4pVPP!V3/T;'g/BwrM+V'd$\!V4'dVR 8wd \R 4hRp V'g`V'gX\V\"4'dB\$P&P)V4'g\+WSR VR 7p RpV P,pWYV3# \TP3dLi;i) zFile handling for PyArrow.Nz pyarrow.fsignore)errorsfsspecz8storage_options not supported with a pyarrow FileSystem.z9filesystem must be a pyarrow or fsspec FileSystem, not a r*rbz8storage_options passed with buffer, or non-supported URLFis_textr8)rr isinstance FileSystemNotImplementedErrorspecAbstractFileSystemr/type__name__rfrom_uri TypeError ArrowInvalidcore url_to_fsrrosr6isdirrhandle) r6r7r8r:r;path_or_handlepa_fsr@pahandless &&&&& r$_get_path_or_handlerWVs$D)N ~*<I+HXF  B0@0@!A!A)N Jr;;3Q3Q$R$R b**+- ^$$  "+I6B.|%>t%D" :/9F!'!6!6"#2#8#8b" B &"8"8DDLSTTG  ~s + + n--  %   B &&7r/  sGG G cH]tRt^t]RRl4tRRltR RRlltRtR#) r!c V^8dQhRRRR/#rdfrr Noner")r#s"r$r%BaseImpl.__annotate__sLLyLTLr&c H\V\4'g \R4hR#)z+to_parquet only supports IO with DataFramesN)rDrr/)r[s&r$validate_dataframeBaseImpl.validate_dataframes "i((JK K)r&c V^8dQhRRRR/#rZr")r#s"r$r%r]s(( (4(r&c \V4hNr )selfr[r6 compressionkwargss&&&&,r$writeBaseImpl.write !$''r&NcV^8dQhRR/#)rr rr")r#s"r$r%r]s((I(r&c \V4hrcrd)rer6columnsrgs&&&,r$read BaseImpl.readrjr&r"rc) rJ __module__ __qualname____firstlineno__ staticmethodr_rhrn__static_attributes__r"r&r$r!r!s%LL(((r&r!cb]tRt^tRRltR RRlltRR]PRRR3RRlltRt R#) r,cV^8dQhRR/#rr r\r")r#s"r$r%PyArrowImpl.__annotate__s  $ r&c <\RRR7^RIp^RIpWnR#)r*z(pyarrow is required for parquet support.extraN)r pyarrow.parquet(pandas.core.arrays.arrow.extension_typesapi)rer*pandass& r$__init__PyArrowImpl.__init__s" G   8r&Nc4V^8dQhRRRRRRRRR R R R R R/#)rr[rr6zFilePath | WriteBuffer[bytes]rfrindex bool | Noner8r9partition_colslist[str] | Noner r\r")r#s"r$r%rxsY@ @ @ ,@ / @  @ / @ )@  @ r&c VPV4RVPRR4/p VeWIR&VPPP!V3/V Bp VP 'dPR\ P!VP 4/p V PPp /V CV Cp V PV 4p \VVVRVRJR7wrp\V\P4'd\VR4'dn\VP \"\$34'dH\VP \$4'dVP P'4pM VP pVe0VPP(P*!V V3RVRVR V/VBM,VPP(P,!V V3RVR V/VBVeVP/4R#R# TeTP/4ii;i) schemaNpreserve_index PANDAS_ATTRSwb)r8r:r;namerfr filesystem)r_popr~Table from_pandasattrsjsondumpsrmetadatareplace_schema_metadatarWrDioBufferedWriterhasattrrrbytesdecodeparquetwrite_to_dataset write_tableclose)rer[r6rfrr8rrrgfrom_pandas_kwargstable df_metadataexisting_metadatamerged_metadatarSrVs&&&&&&&&, r$rhPyArrowImpl.writes #.6 8T8R-S  38/ 0**2D1CD 888)4::bhh+?@K % 5 5 B!2BkBO11/BE.A  +!- / + ~r'8'8 9 9//>..e ==.--u55!/!4!4!;!;!=!/!4!4 )  11"!,$2  *     ,,"!, *   " #w" #s +AG""G9c(V^8dQhRRRRRRRR/#) r dtype_backendDtypeBackend | lib.NoDefaultr8r9to_pandas_kwargszdict[str, Any] | Noner rr")r#s"r$r%rxs4. . 4 . / . 0.  . r&c fRVR&\VVVRR7wrpVPPP!V 3RVRVRV/VBp \ 4;_uu_4\ RR\ 4\V VVR 7p R R R 4V PP'dTR V PP9d9V PPR ,p \P!V 4X n X V eV P4## +'giL;i T eT P4ii;i) Tuse_pandas_metadatarA)r8r:rmrfiltersr>make_block is deprecated)rrNs PANDAS_ATTRS)rWr~r read_tablerrr rrrrloadsrr)rer6rmrrr8rrrgrSrVpa_tableresultrs&&&&&&&&, r$rnPyArrowImpl.reads')-$%.A  + / +  xx''22&   H !!." /"/%5 "'''"hoo&>&>>"*//":":?"KK#'::k#:FL" #%"!$" #s*?D D8#DAD D DD0r~snappyNNNN) rJrprqrrrrhr no_defaultrnrtr"r&r$r,r,s0 @ J69nn1526. . r&r,cB]tRtRtRRltR RRlltR RRlltR tR#) r-i!cV^8dQhRR/#rwr")r#s"r$r%FastParquetImpl.__annotate__"s$r&c ,\RRR7pWnR#)r+z,fastparquet is required for parquet support.rzN)r r~)rer+s& r$rFastParquetImpl.__init__"s1 !O r&Nc(V^8dQhRRRRRRRR/#) rr[rrfz*Literal['snappy', 'gzip', 'brotli'] | Noner8r9r r\r")r#s"r$r%r*s333 3@ 3/3 3r&c aa VPV4RV9dVe \R4hRV9dVPR4pVeRVR&Ve \R4h\ V4p\ V4'd\ R4o V V3RlVR&MS'd \R 4h\R R 7;_uu_4VPP!VV3R VR VRV/VBRRR4R# +'giR#;i) partition_onNzYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datahive file_scheme9filesystem is not implemented for the fastparquet engine.r@c\<SP!VR3/S;'g/BP4#)r)open)r6_r@r8s&&r$'FastParquetImpl.write..Ms/&++d3.44"3df3r& open_withz?storage_options passed with file object or non-fsspec file pathT)recordrf write_index) r_r/rrFrrr rr~rh) rer[r6rfrrr8rrgr@s &&&&&&f&,@r$rhFastParquetImpl.write*s  # V #(BK  V ##ZZ7N  %$*F= !  !%K  d#   /9F#F; Q 4 ( ( HHNN ( "   ,     ) ( ( (s 3%C"" C3 c$V^8dQhRRRRRR/#)rr8r9r dict | Noner rr")r#s"r$r%r_s*7 7 / 7 &7  7 r&c /pVPR\P4p RVR&V \PJd \R4hVe \ R4hVe \ R4h\ V4pRp \ V4'd8\R4p V P!VR3/T;'g/BPVR &MV\V\4'dA\PPV4'g\VRRVR 7p V P pVP"P$!V3/VBp \'4;_uu_4\)R R \*4V P,!RR VRV/VBuuRRR4V eV P/4## +'giM;iT eT P/4R#R# T eT P/4ii;i)rF pandas_nullszHThe 'dtype_backend' argument is not supported for the fastparquet engineNrz?to_pandas_kwargs is not implemented for the fastparquet engine.r@rAr7rBr>rrmrr")rrrr/rFrrr rr7rDrrPr6rQrrRr~ ParquetFilerrr to_pandasr) rer6rmrr8rrrgparquet_kwargsrrVr@ parquet_files &&&&&&&, r$rnFastParquetImpl.read_s*, ?CNNC ).~&  .%   !%K   '%Q d#   /9F#);;tT#Uo>S>SQS#U#X#XN4 c " "277==+>+>!dE?G>>D 88//GGL!!." $--#-48> "!" #"!!" #w" #s$/F3'F' F3 F F33G rr)NNNNN)rJrprqrrrrhrnrtr"r&r$r-r-!s3j7 7 r&r-r(c<V^8dQhRRRRRRRRR R R R R RRRRR/ #)rr[rr6z$FilePath | WriteBuffer[bytes] | Nonerrrfrrrr8r9rrrrr z bytes | Noner")r#s"r$r%r%sm``` .` `+ `  ` + `%```r&c ,\V\4'dV.p\V4p Vf\P!4MTp V P !VV 3RVRVRVRVRV/VBVf3\V \P4'gQhV P 4#R#)a Write a DataFrame to the parquet format. Parameters ---------- df : DataFrame path : str, path object, file-like object, or None, default None String, path object (implementing ``os.PathLike[str]``), or file-like object implementing a binary ``write()`` function. If None, the result is returned as bytes. If a string, it will be used as Root Directory path when writing a partitioned dataset. The engine fastparquet does not accept file-like objects. engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto' Parquet library to use. If 'auto', then the option ``io.parquet.engine`` is used. The default ``io.parquet.engine`` behavior is to try 'pyarrow', falling back to 'fastparquet' if 'pyarrow' is unavailable. When using the ``'pyarrow'`` engine and no storage options are provided and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first. Use the filesystem keyword with an instantiated fsspec filesystem if you wish to use its implementation. compression : {'snappy', 'gzip', 'brotli', 'lz4', 'zstd', None}, default 'snappy'. Name of the compression to use. Use ``None`` for no compression. index : bool, default None If ``True``, include the dataframe's index(es) in the file output. If ``False``, they will not be written to the file. If ``None``, similar to ``True`` the dataframe's index(es) will be saved. However, instead of being saved as values, the RangeIndex will be stored as a range in the metadata so it doesn't require much space and is faster. Other indexes will be included as columns in the file output. partition_cols : str or list, optional, default None Column names by which to partition the dataset. Columns are partitioned in the order they are given. Must be None if path is not a string. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to ``urllib.request.Request`` as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more details, and for more examples on storage options refer `here `_. filesystem : fsspec or pyarrow filesystem, default None Filesystem object to use when reading the parquet file. Only implemented for ``engine="pyarrow"``. .. versionadded:: 2.1.0 **kwargs Additional keyword arguments passed to the engine: * For ``engine="pyarrow"``: passed to :func:`pyarrow.parquet.write_table` or :func:`pyarrow.parquet.write_to_dataset` (when using partition_cols) * For ``engine="fastparquet"``: passed to :func:`fastparquet.write` Returns ------- bytes if no path argument is provided else None Nrfrrr8r)rDrr4rBytesIOrhgetvalue) r[r6rrfrr8rrrgimpl path_or_bufs &&&&&&&&, r$ to_parquetrsV.#&&() f DAESWKJJ      &  (     |+rzz2222##%%r&rc<V^8dQhRRRRRRRRR R R R R RRRRR/ #)rr6zFilePath | ReadBuffer[bytes]rrrmrr8r9rrrrrz&list[tuple] | list[list[tuple]] | Nonerrr rr")r#s"r$r%r%smkk &k kk+ k 0 k  k4k"kkr&c l\V4p \V4V P!V3RVRVRVRVRVRV/VB#)aM Load a parquet object from the file path, returning a DataFrame. The function automatically handles reading the data from a parquet file and creates a DataFrame with the appropriate structure. Parameters ---------- path : str, path object or file-like object String, path object (implementing ``os.PathLike[str]``), or file-like object implementing a binary ``read()`` function. The string could be a URL. Valid URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is expected. A local file could be: ``file://localhost/path/to/table.parquet``. A file URL can also be a path to a directory that contains multiple partitioned parquet files. Both pyarrow and fastparquet support paths to directories as well as file URLs. A directory path could be: ``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``. engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto' Parquet library to use. If 'auto', then the option ``io.parquet.engine`` is used. The default ``io.parquet.engine`` behavior is to try 'pyarrow', falling back to 'fastparquet' if 'pyarrow' is unavailable. When using the ``'pyarrow'`` engine and no storage options are provided and a filesystem is implemented by both ``pyarrow.fs`` and ``fsspec`` (e.g. "s3://"), then the ``pyarrow.fs`` filesystem is attempted first. Use the filesystem keyword with an instantiated fsspec filesystem if you wish to use its implementation. columns : list, default=None If not None, only these columns will be read from the file. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to ``urllib.request.Request`` as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more details, and for more examples on storage options refer `here `_. dtype_backend : {'numpy_nullable', 'pyarrow'} Back-end data type applied to the resultant :class:`DataFrame` (still experimental). If not specified, the default behavior is to not use nullable data types. If specified, the behavior is as follows: * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame` * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype` :class:`DataFrame` .. versionadded:: 2.0 filesystem : fsspec or pyarrow filesystem, default None Filesystem object to use when reading the parquet file. Only implemented for ``engine="pyarrow"``. .. versionadded:: 2.1.0 filters : List[Tuple] or List[List[Tuple]], default None To filter out data. Filter syntax: [[(column, op, val), ...],...] where op is [==, =, >, >=, <, <=, !=, in, not in] The innermost tuples are transposed into a set of filters applied through an `AND` operation. The outer list combines these sets of filters through an `OR` operation. A single list of tuples can also be used, meaning that no `OR` operation between set of filters is to be conducted. Using this argument will NOT result in row-wise filtering of the final partitions unless ``engine="pyarrow"`` is also specified. For other engines, filtering is only performed at the partition level, that is, to prevent the loading of some row-groups and/or files. .. versionadded:: 2.1.0 to_pandas_kwargs : dict | None, default None Keyword arguments to pass through to :func:`pyarrow.Table.to_pandas` when ``engine="pyarrow"``. .. versionadded:: 3.0.0 **kwargs Additional keyword arguments passed to the engine: * For ``engine="pyarrow"``: passed to :func:`pyarrow.parquet.read_table` * For ``engine="fastparquet"``: passed to :meth:`fastparquet.ParquetFile.to_pandas` Returns ------- DataFrame DataFrame based on parquet file. See Also -------- DataFrame.to_parquet : Create a parquet object that serializes a DataFrame. Examples -------- >>> original_df = pd.DataFrame({"foo": range(5), "bar": range(5, 10)}) >>> original_df foo bar 0 0 5 1 1 6 2 2 7 3 3 8 4 4 9 >>> df_parquet_bytes = original_df.to_parquet() >>> from io import BytesIO >>> restored_df = pd.read_parquet(BytesIO(df_parquet_bytes)) >>> restored_df foo bar 0 0 5 1 1 6 2 2 7 3 3 8 4 4 9 >>> restored_df.equals(original_df) True >>> restored_bar = pd.read_parquet(BytesIO(df_parquet_bytes), columns=["bar"]) >>> restored_bar bar 0 5 1 6 2 7 3 8 4 9 >>> restored_bar.equals(original_df[["bar"]]) True The function uses `kwargs` that are passed directly to the engine. In the following example, we use the `filters` argument of the pyarrow engine to filter the rows of the DataFrame. Since `pyarrow` is the default engine, we can omit the `engine` argument. Note that the `filters` argument is implemented by the `pyarrow` engine, which can benefit from multithreading and also potentially be more economical in terms of memory. >>> sel = [("foo", ">", 2)] >>> restored_part = pd.read_parquet(BytesIO(df_parquet_bytes), filters=sel) >>> restored_part foo bar 0 3 8 1 4 9 rmrr8rrr)r4r rn) r6rrmr8rrrrrgrs &&&&&&&&, r$ read_parquetrso@ f D & 99    (  $   *    r&)NrAF)Nr(rNNNN)2__doc__ __future__rrrrPtypingrrrwarningsrr pandas._libsrpandas.compat._optionalr pandas.errorsr r pandas.util._decoratorsr pandas.util._validatorsr rrrpandas.io._utilrpandas.io.commonrrrrrpandas._typingrrrrrrr4rWr!r,r-rrrr"r&r$rs"   >/7 2GD<'~ ( (| (| ~u hu p`F H $-125..6:$(kkr&