U -e%@sXddlmZddlmZddlmZddlmZmZddl Z ddl Z ddl m Z ddl ZddlZddlZddlZddlZddlZddlZddlmZzddlmZWn8ek rZzedeeddW5dZ[XYnXdd lmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-dd lm.Z/dd l0m1Z1m2Z2m3Z3m4Z4d Z5ddZ6ddZ7ddZ8dnddZ9dZ:ddZ;e4dde;de<Z=GdddZ>e?dZ@dd ZAd!d"ZBd#d$ZCd%ZDd&ZEGd'd(d(ZFd)d*ZGGd+d,d,ZHGd-d.d.ZIGd/d0d0ZJGd1d2d2ZKd3d4ZLd5d6ZMd7d8ZNd9hZOGd:d;d;ZPdodZRd?ZSd@ZTGdAdBdBZUdpdEdFZVdGdHZWGdIdJdJZXdKZYdLZZdddddMddMddNdddMdddddddOdPdQZ[eY\dRdS]dTeSfdUe:eZe[_^dqdVdWZ_eY\dXdS]eSdYfdZe:d[e__^drd_d`Z`daZadb\eDeae`_^dcddZbdsdedfZcdtdgdhZddudidjZedvdkdlZfdmZgdS)w) defaultdict)futures) nullcontext)partialreduceN) CollectionzPThe pyarrow installation is not built with support for the Parquet file format ()) ParquetReader Statistics FileMetaDataRowGroupMetaDataColumnChunkMetaData ParquetSchema ColumnSchemaParquetLogicalTypeFileEncryptionPropertiesFileDecryptionProperties)LocalFileSystem FileSystemFileType_resolve_filesystem_and_path_ensure_filesystem filesystem)guid _is_path_like_stringify_path_deprecate_api)ZhdfscCs,t|}tj|}|jtkr$|jS|SdSN)rurllibparseurlparsescheme_URI_STRIP_SCHEMESpath)r$Z parsed_urir%U/var/www/html/Darija-Ai-Train/env/lib/python3.8/site-packages/pyarrow/parquet/core.py _parse_uri<s   r'cCs2|dkrt||St|}t|}||fSdSr)legacyfsresolve_filesystem_and_pathrr')Zpassed_filesystemr$Z parsed_pathr%r%r&_get_filesystem_and_pathGs   r*cCsRt|tr<|D]*}t|tr&td}nd}||krdSqnt|trNd|kSdS)NrTF) isinstancebyteschrstr)valbyteZ compare_tor%r%r&_check_contains_nullPs     r2TcCs|dk rt|dks&tdd|Dr.tdt|ddtrF|g}|r|D]@}|D]6\}}}t|tr|tdd|Dst|rVtdqVqN|S)z+ Check if filters are well-formed. Nrcss|]}t|dkVqdS)rN)len.0fr%r%r& csz!_check_filters..zMalformed filterscss|]}t|VqdSr)r2)r5vr%r%r&r7oszBNull-terminated binary strings are not supported as filter values.) r3any ValueErrorr,r/listallr2NotImplementedError)filterscheck_null_strings conjunctioncolopr0r%r%r&_check_filters^s$rCaPredicates are expressed using an ``Expression`` or using the disjunctive normal form (DNF), like ``[[('x', '=', 0), ...], ...]``. DNF allows arbitrary boolean logical combinations of single column predicates. The innermost tuples each describe a single column predicate. The list of inner predicates is interpreted as a conjunction (AND), forming a more selective and multiple column predicate. Finally, the most outer list combines these filters as a disjunction (OR). Predicates may also be passed as List[Tuple]. This form is interpreted as a single conjunction. To express OR in predicates, one must use the (preferred) List[List[Tuple]] notation. Each tuple has format: (``key``, ``op``, ``value``) and compares the ``key`` with the ``value``. The supported ``op`` are: ``=`` or ``==``, ``!=``, ``<``, ``>``, ``<=``, ``>=``, ``in`` and ``not in``. If the ``op`` is ``in`` or ``not in``, the ``value`` must be a collection such as a ``list``, a ``set`` or a ``tuple``. Examples: Using the ``Expression`` API: .. code-block:: python import pyarrow.compute as pc pc.field('x') = 0 pc.field('y').isin(['a', 'b', 'c']) ~pc.field('y').isin({'a', 'b'}) Using the DNF format: .. code-block:: python ('x', '=', 0) ('y', 'in', ['a', 'b', 'c']) ('z', 'not in', {'a','b'}) csrddlmt|jr|St|dd}fddg}|D](}fdd|D}|ttj|q>> filters_to_expression([('foo', '==', 'bar')]) Returns ------- pyarrow.compute.Expression An Expression representing the filters rNF)r?cs|}|dks|dkr"||kS|dkr2||kS|dkrB||kS|dkrR||kS|dkrb||kS|dkrr||kS|dkr||S|d kr||Std |||fdS) N===!=<><=>=innot inz,"{0}" is not a valid operator in predicates.)fieldisinr:format)rArBr0rM)dsr%r&convert_single_predicates,   z7filters_to_expression..convert_single_predicatecsg|]\}}}|||qSr%r%)r5rArBr0)rQr%r& sz)filters_to_expression..) pyarrow.datasetdatasetr, ExpressionrCappendroperatorand_or_)r>Zdisjunction_membersr@Zconjunction_membersr%)rQrPr&filters_to_expressions     rZ_filters_to_expressionz10.0.0c @seZdZdZdddddddddddd ddZdd Zd d Zd d ZeddZ eddZ eddZ eddZ d*e dddZee dddZd+ddZd,dd Zd-d"d#Zd.d$d%Zd/d&d'Zd0d(d)ZdS)1 ParquetFilea  Reader interface for a single Parquet file. Parameters ---------- source : str, pathlib.Path, pyarrow.NativeFile, or file-like object Readable source. For passing bytes or buffer-like file containing a Parquet file, use pyarrow.BufferReader. metadata : FileMetaData, default None Use existing metadata object, rather than reading from file. common_metadata : FileMetaData, default None Will be used in reads for pandas schema metadata if not found in the main file's metadata, no other uses at the moment. read_dictionary : list List of column names to read directly as DictionaryArray. memory_map : bool, default False If the source is a file path, use a memory map to read file, which can improve performance in some environments. buffer_size : int, default 0 If positive, perform read buffering when deserializing individual column chunks. Otherwise IO calls are unbuffered. pre_buffer : bool, default False Coalesce and issue file reads in parallel to improve performance on high-latency filesystems (e.g. S3). If True, Arrow will use a background I/O thread pool. coerce_int96_timestamp_unit : str, default None Cast timestamps that are stored in INT96 format to a particular resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' and therefore INT96 timestamps will be inferred as timestamps in nanoseconds. decryption_properties : FileDecryptionProperties, default None File decryption properties for Parquet Modular Encryption. thrift_string_size_limit : int, default None If not None, override the maximum total string size allocated when decoding Thrift structures. The default limit should be sufficient for most Parquet files. thrift_container_size_limit : int, default None If not None, override the maximum total size of containers allocated when decoding Thrift structures. The default limit should be sufficient for most Parquet files. filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise it will be parsed as an URI to determine the filesystem. Examples -------- Generate an example PyArrow Table and write it to Parquet file: >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') Create a ``ParquetFile`` object from the Parquet file: >>> parquet_file = pq.ParquetFile('example.parquet') Read the data: >>> parquet_file.read() pyarrow.Table n_legs: int64 animal: string ---- n_legs: [[2,2,4,4,5,100]] animal: [["Flamingo","Parrot","Dog","Horse","Brittle stars","Centipede"]] Create a ParquetFile object with "animal" column as DictionaryArray: >>> parquet_file = pq.ParquetFile('example.parquet', ... read_dictionary=["animal"]) >>> parquet_file.read() pyarrow.Table n_legs: int64 animal: dictionary ---- n_legs: [[2,2,4,4,5,100]] animal: [ -- dictionary: ["Flamingo","Parrot",...,"Brittle stars","Centipede"] -- indices: [0,1,2,3,4,5]] NFr) metadatacommon_metadataread_dictionary memory_map buffer_size pre_buffercoerce_int96_timestamp_unitdecryption_propertiesthrift_string_size_limitthrift_container_size_limitrc Csrt|dd|_t|| |\} }| dk r6| |}d|_t|_|jj|||||||| | | d ||_||_ dS)NclosedT) Zuse_memory_maprarbr_r]rcrdrerf) getattr _close_sourceropen_input_filer readeropenr^_build_nested_paths_nested_paths_by_prefix) selfsourcer]r^r_r`rarbrcrdrerfrr%r%r&__init__Fs0  zParquetFile.__init__cCs|Srr%ror%r%r& __enter__aszParquetFile.__enter__cOs |dSrcloseroargskwargsr%r%r&__exit__dszParquetFile.__exit__cCsn|jj}tt}t|D]P\}}|d}|dd}||||sHqd||df}|dd}q4q|S)Nr.)rkZ column_pathsrr; enumeraterVjoin)ropathsresultir$keyrestr%r%r&rmgs zParquetFile._build_nested_pathscCs|jjS)z. Return the Parquet metadata. )rkr]rrr%r%r&r]zszParquetFile.metadatacCs|jjS)zG Return the Parquet schema, unconverted to Arrow types )r]schemarrr%r%r&rszParquetFile.schemacCs|jjS)a Return the inferred Arrow schema, converted from the whole Parquet file's schema Examples -------- Generate an example Parquet file: >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') >>> parquet_file = pq.ParquetFile('example.parquet') Read the Arrow schema: >>> parquet_file.schema_arrow n_legs: int64 animal: string )rk schema_arrowrrr%r%r&rszParquetFile.schema_arrowcCs|jjS)a. Return the number of row groups of the Parquet file. Examples -------- >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') >>> parquet_file = pq.ParquetFile('example.parquet') >>> parquet_file.num_row_groups 1 )rknum_row_groupsrrr%r%r&rszParquetFile.num_row_groupsforcecCs|js |r|jdSr)rirkru)rorr%r%r&rus zParquetFile.close)returncCs|jjSr)rkrgrrr%r%r&rgszParquetFile.closedTcCs |j||d}|jj|||dS)a Read a single row group from a Parquet file. Parameters ---------- i : int Index of the individual row group that we want to read. columns : list If not None, only these columns will be read from the row group. A column name may be a prefix of a nested field, e.g. 'a' will select 'a.b', 'a.c', and 'a.d.e'. use_threads : bool, default True Perform multi-threaded column reads. use_pandas_metadata : bool, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded. Returns ------- pyarrow.table.Table Content of the row group as a table (of columns) Examples -------- >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') >>> parquet_file = pq.ParquetFile('example.parquet') >>> parquet_file.read_row_group(0) pyarrow.Table n_legs: int64 animal: string ---- n_legs: [[2,2,4,4,5,100]] animal: [["Flamingo","Parrot",...,"Brittle stars","Centipede"]] use_pandas_metadatacolumn_indices use_threads)_get_column_indicesrkread_row_group)rorcolumnsrrrr%r%r&rs* zParquetFile.read_row_groupcCs |j||d}|jj|||dS)a Read a multiple row groups from a Parquet file. Parameters ---------- row_groups : list Only these row groups will be read from the file. columns : list If not None, only these columns will be read from the row group. A column name may be a prefix of a nested field, e.g. 'a' will select 'a.b', 'a.c', and 'a.d.e'. use_threads : bool, default True Perform multi-threaded column reads. use_pandas_metadata : bool, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded. Returns ------- pyarrow.table.Table Content of the row groups as a table (of columns). Examples -------- >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') >>> parquet_file = pq.ParquetFile('example.parquet') >>> parquet_file.read_row_groups([0,0]) pyarrow.Table n_legs: int64 animal: string ---- n_legs: [[2,2,4,4,5,...,2,4,4,5,100]] animal: [["Flamingo","Parrot","Dog",...,"Brittle stars","Centipede"]] rr)rrkread_row_groups)ro row_groupsrrrrr%r%r&rs*zParquetFile.read_row_groupscCs<|dkrtd|jj}|j||d}|jj||||d}|S)a Read streaming batches from a Parquet file. Parameters ---------- batch_size : int, default 64K Maximum number of records to yield per batch. Batches may be smaller if there aren't enough rows in the file. row_groups : list Only these row groups will be read from the file. columns : list If not None, only these columns will be read from the file. A column name may be a prefix of a nested field, e.g. 'a' will select 'a.b', 'a.c', and 'a.d.e'. use_threads : boolean, default True Perform multi-threaded column reads. use_pandas_metadata : boolean, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded. Yields ------ pyarrow.RecordBatch Contents of each batch as a record batch Examples -------- Generate an example Parquet file: >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') >>> parquet_file = pq.ParquetFile('example.parquet') >>> for i in parquet_file.iter_batches(): ... print("RecordBatch") ... print(i.to_pandas()) ... RecordBatch n_legs animal 0 2 Flamingo 1 2 Parrot 2 4 Dog 3 4 Horse 4 5 Brittle stars 5 100 Centipede Nrr)rrr)ranger]rrrk iter_batches)ro batch_sizerrrrrZbatchesr%r%r&rs3zParquetFile.iter_batchescCs|j||d}|jj||dS)aW Read a Table from Parquet format. Parameters ---------- columns : list If not None, only these columns will be read from the file. A column name may be a prefix of a nested field, e.g. 'a' will select 'a.b', 'a.c', and 'a.d.e'. use_threads : bool, default True Perform multi-threaded column reads. use_pandas_metadata : bool, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded. Returns ------- pyarrow.table.Table Content of the file as a table (of columns). Examples -------- Generate an example Parquet file: >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') >>> parquet_file = pq.ParquetFile('example.parquet') Read a Table: >>> parquet_file.read(columns=["animal"]) pyarrow.Table animal: string ---- animal: [["Flamingo","Parrot",...,"Brittle stars","Centipede"]] rr)rrkZread_all)rorrrrr%r%r&read[s)zParquetFile.readcCs||}|jj||dS)a  Read contents of file for the given columns and batch size. Notes ----- This function's primary purpose is benchmarking. The scan is executed on a single thread. Parameters ---------- columns : list of integers, default None Select columns to read, if None scan all columns. batch_size : int, default 64K Number of rows to read at a time internally. Returns ------- num_rows : int Number of rows in file Examples -------- >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') >>> parquet_file = pq.ParquetFile('example.parquet') >>> parquet_file.scan_contents() 6 )r)rrk scan_contents)rorrrr%r%r&rs" zParquetFile.scan_contentscs|dkr dSg}|D]}|jkr|j|q|rjj}jdk rRjjnd}|rld|krlt|}n|rd|krt|}ng}|dk r|r|fdd|D7}|S)Npandascs"g|]}t|tsj|qSr%)r,dictrkZcolumn_name_idx)r5descrrrr%r&rRs z3ParquetFile._get_column_indices..)rnextendr]r^_get_pandas_index_columns)roZ column_namesrindicesnameZfile_keyvaluesZcommon_keyvalues index_columnsr%rrr&rs,        zParquetFile._get_column_indices)F)NTF)NTF)rNNTF)NTF)Nr)F)__name__ __module__ __qualname____doc__rqrsryrmpropertyr]rrrboolrurgrrrrrrr%r%r%r&r\sNW      / 0 > . &r\z [ ,;{}() =]cCs td|S)N_)_SPARK_DISALLOWED_CHARSsub)rr%r%r&_sanitized_spark_field_namesrc Csd|krxg}d}|D]J}|j}t|}||krTd}t||j|j|j}||q||qtj||jd}||fS|dfSdS)NsparkFT)r]) rrparMtypeZnullabler]rVr) rflavorZsanitized_fieldsschema_changedrMrZsanitized_nameZsanitized_field new_schemar%r%r&_sanitize_schemas"   rcs8d|kr0fddtjD}tjj||dSSdS)Nrcsg|] }|qSr%r%)r5rtabler%r&rRsz#_sanitize_table..)r)rZ num_columnsrTable from_arrays)rrrZ column_datar%rr&_sanitize_tablesraversion : {"1.0", "2.4", "2.6"}, default "2.6" Determine which Parquet logical types are available for use, whether the reduced set from the Parquet 1.x.x format or the expanded logical types added in later format versions. Files written with version='2.4' or '2.6' may not be readable in all Parquet implementations, so version='1.0' is likely the choice that maximizes file compatibility. UINT32 and some logical types are only available with version '2.4'. Nanosecond timestamps are only available with version '2.6'. Other features such as compression algorithms or the new serialized data page format must be enabled separately (see 'compression' and 'data_page_version'). use_dictionary : bool or list Specify if we should use dictionary encoding in general or only for some columns. compression : str or dict Specify the compression codec, either on a general basis or per-column. Valid values: {'NONE', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'ZSTD'}. write_statistics : bool or list Specify if we should write statistics in general (default is True) or only for some columns. use_deprecated_int96_timestamps : bool, default None Write timestamps to INT96 Parquet format. Defaults to False unless enabled by flavor argument. This take priority over the coerce_timestamps option. coerce_timestamps : str, default None Cast timestamps to a particular resolution. If omitted, defaults are chosen depending on `version`. By default, for ``version='1.0'`` (the default) and ``version='2.4'``, nanoseconds are cast to microseconds ('us'), while for other `version` values, they are written natively without loss of resolution. Seconds are always cast to milliseconds ('ms') by default, as Parquet does not have any temporal type with seconds resolution. If the casting results in loss of data, it will raise an exception unless ``allow_truncated_timestamps=True`` is given. Valid values: {None, 'ms', 'us'} allow_truncated_timestamps : bool, default False Allow loss of data when coercing timestamps to a particular resolution. E.g. if microsecond or nanosecond data is lost when coercing to 'ms', do not raise an exception. Passing ``allow_truncated_timestamp=True`` will NOT result in the truncation exception being ignored unless ``coerce_timestamps`` is not None. data_page_size : int, default None Set a target threshold for the approximate encoded size of data pages within a column chunk (in bytes). If None, use the default data page size of 1MByte. flavor : {'spark'}, default None Sanitize schema or set other compatibility options to work with various target systems. filesystem : FileSystem, default None If nothing passed, will be inferred from `where` if path-like, else `where` is already a file-like object so no filesystem is needed. compression_level : int or dict, default None Specify the compression level for a codec, either on a general basis or per-column. If None is passed, arrow selects the compression level for the compression codec in use. The compression level has a different meaning for each codec, so you have to read the documentation of the codec you are using. An exception is thrown if the compression codec does not allow specifying a compression level. use_byte_stream_split : bool or list, default False Specify if the byte_stream_split encoding should be used in general or only for some columns. If both dictionary and byte_stream_stream are enabled, then dictionary is preferred. The byte_stream_split encoding is valid only for floating-point data types and should be combined with a compression codec. column_encoding : string or dict, default None Specify the encoding scheme on a per column basis. Currently supported values: {'PLAIN', 'BYTE_STREAM_SPLIT'}. Certain encodings are only compatible with certain data types. Please refer to the encodings section of `Reading and writing Parquet files `_. data_page_version : {"1.0", "2.0"}, default "1.0" The serialized Parquet data page format version to write, defaults to 1.0. This does not impact the file schema logical types and Arrow to Parquet type casting behavior; for that use the "version" option. use_compliant_nested_type : bool, default True Whether to write compliant Parquet nested type (lists) as defined `here `_, defaults to ``False``. For ``use_compliant_nested_type=True``, this will write into a list with 3-level structure where the middle level, named ``list``, is a repeated group with a single field named ``element``:: group (LIST) { repeated group list { element; } } For ``use_compliant_nested_type=False``, this will also write into a list with 3-level structure, where the name of the single field of the middle level ``list`` is taken from the element name for nested columns in Arrow, which defaults to ``item``:: group (LIST) { repeated group list { item; } } encryption_properties : FileEncryptionProperties, default None File encryption properties for Parquet Modular Encryption. If None, no encryption will be done. The encryption properties can be created using: ``CryptoFactory.file_encryption_properties()``. write_batch_size : int, default None Number of values to write to a page at a time. If None, use the default of 1024. ``write_batch_size`` is complementary to ``data_page_size``. If pages are exceeding the ``data_page_size`` due to large column values, lowering the batch size can help keep page sizes closer to the intended size. dictionary_pagesize_limit : int, default None Specify the dictionary page size limit per row group. If None, use the default 1MB. store_schema : bool, default True By default, the Arrow schema is serialized and stored in the Parquet file metadata (in the "ARROW:schema" key). When reading the file, if this key is available, it will be used to more faithfully recreate the original Arrow data. For example, for tz-aware timestamp columns it will restore the timezone (Parquet only stores the UTC values without timezone), or columns with duration type will be restored from the int64 Parquet column. write_page_index : bool, default False Whether to write a page index in general for all columns. Writing statistics to the page index disables the old method of writing statistics to each data page header. The page index makes statistics-based filtering more efficient than the page header, as it gathers all the statistics for a Parquet file in a single place, avoiding scattered I/O. Note that the page index is not yet used on the read size by PyArrow. atGenerate an example PyArrow Table and RecordBatch: >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> batch = pa.record_batch([[2, 2, 4, 4, 5, 100], ... ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]], ... names=['n_legs', 'animal']) create a ParquetWriter object: >>> import pyarrow.parquet as pq >>> writer = pq.ParquetWriter('example.parquet', table.schema) and write the Table into the Parquet file: >>> writer.write_table(table) >>> writer.close() >>> pq.read_table('example.parquet').to_pandas() n_legs animal 0 2 Flamingo 1 2 Parrot 2 4 Dog 3 4 Horse 4 5 Brittle stars 5 100 Centipede create a ParquetWriter object for the RecordBatch: >>> writer2 = pq.ParquetWriter('example2.parquet', batch.schema) and write the RecordBatch into the Parquet file: >>> writer2.write_batch(batch) >>> writer2.close() >>> pq.read_table('example2.parquet').to_pandas() n_legs animal 0 2 Flamingo 1 2 Parrot 2 4 Dog 3 4 Horse 4 5 Brittle stars 5 100 Centipede c@s`eZdZdeeZddd Zd d Zd d Z ddZ dddZ dddZ dddZ ddZdS) ParquetWritera Class for incrementally building a Parquet file for Arrow tables. Parameters ---------- where : path or file-like object schema : pyarrow.Schema {} writer_engine_version : unused **options : dict If options contains a key `metadata_collector` then the corresponding value is assumed to be a list (or any object with `.append` method) that will be filled with the file metadata instance of the written file. Examples -------- {} N2.6TsnappyF1.0cKs| dkr"|dk rd|krd} nd} ||_|dk rBt||\}|_nd|_||_||_d|_t||dd\}}|dk rt|tj r| |d}|_q|j |dd}|_n|}| dd|_ d}tj||f||||| | | | ||||||||d ||_d|_dS) NrTF)Zallow_legacy_filesystemwb) compressionmetadata_collectorZV2)versionruse_dictionarywrite_statisticsuse_deprecated_int96_timestampscompression_leveluse_byte_stream_splitcolumn_encodingwriter_engine_versiondata_page_versionuse_compliant_nested_typeencryption_propertieswrite_batch_sizedictionary_pagesize_limit store_schemawrite_page_index)rrrrwhere file_handlerr,r(rrlopen_output_streampop_metadata_collector_parquetrwriteris_open)rorrrrrrrrrrrrrrrrrrrroptionsr$ZsinkZengine_versionr%r%r&rqsd  zParquetWriter.__init__cCst|ddr|dS)NrF)rhrurrr%r%r&__del__ s zParquetWriter.__del__cCs|Srr%rrr%r%r&rsszParquetWriter.__enter__cOs |dSNFrtrvr%r%r&ryszParquetWriter.__exit__cCsDt|tjr|||n&t|tjr4|||n tt|dS)a{ Write RecordBatch or Table to the Parquet file. Parameters ---------- table_or_batch : {RecordBatch, Table} row_group_size : int, default None Maximum number of rows in each written row group. If None, the row group size will be the minimum of the input table or batch length and 1024 * 1024. N)r,rZ RecordBatch write_batchr write_table TypeErrorr)roZtable_or_batchrow_group_sizer%r%r&writes  zParquetWriter.writecCs"tj|g|j}|||dS)a Write RecordBatch to the Parquet file. Parameters ---------- batch : RecordBatch row_group_size : int, default None Maximum number of rows in written row group. If None, the row group size will be the minimum of the RecordBatch size and 1024 * 1024. If set larger than 64Mi then 64Mi will be used instead. N)rrZ from_batchesrr)robatchrrr%r%r&r,s zParquetWriter.write_batchcCs^|jrt||j|j}|js t|jj|jddsJd|j|j}t||j j ||ddS)a Write Table to the Parquet file. Parameters ---------- table : Table row_group_size : int, default None Maximum number of rows in each written row group. If None, the row group size will be the minimum of the Table size and 1024 * 1024. If set larger than 64Mi then 64Mi will be used instead. FZcheck_metadatazTTable schema does not match schema used to create file: table: {!s} vs. file: {!s}rN) rrrrrAssertionErrorequalsrOr:rr)rorrmsgr%r%r&r<s zParquetWriter.write_tablecCsH|jr0|jd|_|jdk r0|j|jj|jdk rD|jdS)z; Close the connection to the Parquet file. FN)rrrurrVr]rrrr%r%r&ruVs   zParquetWriter.close)NNrTrTNNFNNrTNNNTF)N)N)N)rrrrO_parquet_writer_arg_docs_parquet_writer_example_docrrqrrsryrrrrur%r%r%r&rs< N   rcCst|dddS)Nrutf8r)jsonloadsdecode) keyvaluesr%r%r&rcsrc@seZdZdZeedddddfddZeeedddddfddZd d Z d d Z d dZ ddZ ddZ ddZdddZdS)ParquetDatasetPiecea DEPRECATED: A single chunk of a potentially larger Parquet dataset to read. The arguments will indicate to read either a single row group or all row groups, and whether to add partition keys to the resulting pyarrow.Table. .. deprecated:: 5.0 Directly constructing a ``ParquetDatasetPiece`` is deprecated, as well as accessing the pieces of a ``ParquetDataset`` object. Specify ``use_legacy_dataset=False`` when constructing the ``ParquetDataset`` and use the ``ParquetDataset.fragments`` attribute instead. Parameters ---------- path : str or pathlib.Path Path to file in the file system where this piece is located. open_file_func : callable Function to use for obtaining file handle to dataset piece. file_options : dict Options row_group : int, default None Row group to load. By default, reads all row groups. partition_keys : list of tuples Two-element tuples of ``(column name, ordinal index)``. rbmodeNcCs&tjdtdd||||||dS)Nz^ParquetDatasetPiece is deprecated as of pyarrow 5.0.0 and will be removed in a future version. stacklevel)warningswarn FutureWarning_initror$open_file_func file_options row_grouppartition_keysr%r%r&rqszParquetDatasetPiece.__init__cCs tt}|||||||Sr)r__new__r)r$rrrrror%r%r&_creates zParquetDatasetPiece._createcCs.t||_||_||_|pg|_|p&i|_dSr)rr$rrrrrr%r%r&rs   zParquetDatasetPiece._initcCs2t|tsdS|j|jko0|j|jko0|j|jkSr)r,rr$rrrootherr%r%r&__eq__s    zParquetDatasetPiece.__eq__cCsdt|j|j|j|jS)Nz-{}({!r}, row_group={!r}, partition_keys={!r}))rOrrr$rrrrr%r%r&__repr__s zParquetDatasetPiece.__repr__cCs^d}t|jdkr6ddd|jD}|d|7}||j7}|jdk rZ|d|j7}|S)Nrz, css|]\}}d||VqdS)z{}={}NrO)r5rindexr%r%r&r7sz.ParquetDatasetPiece.__str__..zpartition[{}] z | row_group={})r3rr}rOr$r)rorZ partition_strr%r%r&__str__s   zParquetDatasetPiece.__str__c Cs&|}|jW5QRSQRXdS)z Return the file's metadata. Returns ------- metadata : FileMetaData The file's metadata N)rlr])roZparquetr%r%r& get_metadatas z ParquetDatasetPiece.get_metadatacCs.||j}t|ts$t|f|j}d|_|S)z1 Return instance of ParquetFile. T)rr$r,r\rri)rorkr%r%r&rls   zParquetDatasetPiece.openTFcCs|jdk r|}n(|dk r,t|f|j}nt|jf|j}t|||d}|jdk rf|j|jf|}n |jf|}t |j dkr|dkrt dt |j D]F\} \} } t jt || dd} |j| j} tj| | }|| |}q|jdd|S) a% Read this piece as a pyarrow.Table. Parameters ---------- columns : list of column names, default None use_threads : bool, default True Perform multi-threaded column reads. partitions : ParquetPartitions, default None file : file-like object Passed to ParquetFile. use_pandas_metadata : bool If pandas metadata should be used or not. Returns ------- table : pyarrow.Table The piece as a pyarrow.Table. NrrrrzMust pass partition setsi4)ZdtypeTr)rrlr\rr$rrrrr3rr:r|npfulllevels dictionaryrZDictionaryArrayr append_columnru)rorr partitionsfilerrkrrrrrrr Zarrr%r%r&rs,       zParquetDatasetPiece.read)NTNNF)rrrrrrlrq staticmethodrrrrrrrr%r%r%r&rms,      rc@s:eZdZdZd ddZddZeddZed d ZdS) PartitionSeta A data structure for cataloguing the observed Parquet partitions at a particular level. So if we have /foo=a/bar=0 /foo=a/bar=1 /foo=a/bar=2 /foo=b/bar=0 /foo=b/bar=1 /foo=b/bar=2 Then we have two partition sets, one for foo, another for bar. As we visit levels of the partition hierarchy, a PartitionSet tracks the distinct values and assigns categorical codes to use when reading the pieces Parameters ---------- name : str Name of the partition set. Under which key to collect all values. keys : list All possible values that have been collected for that partition set. NcCs0||_|p g|_ddt|jD|_d|_dS)NcSsi|]\}}||qSr%r%)r5rkr%r%r& 7sz)PartitionSet.__init__..)rkeysr| key_indices _dictionary)rorrr%r%r&rq4s zPartitionSet.__init__cCs<||jkr|j|St|j}|j|||j|<|SdS)z Get the index of the partition value if it is known, otherwise assign one Parameters ---------- key : str or int The value for which we want to known the index. N)rr3rrV)rorrr%r%r& get_index:s     zPartitionSet.get_indexcCsp|jdk r|jSt|jdkr&tdzdd|jD}t|}Wn tk rdt|j}YnX||_|S)NrzNo known partition keyscSsg|] }t|qSr%)intr5xr%r%r&rRVsz+PartitionSet.dictionary..)rr3rr:libarray)roZ integer_keysr r%r%r&r Ls zPartitionSet.dictionarycCst|jt|jkSr)r;rsortedrrr%r%r& is_sorted^szPartitionSet.is_sorted)N) rrrrrqrrr rr%r%r%r&rs  rc@sDeZdZddZddZddZddZd d Zd d Zd dZ dS)ParquetPartitionscCsg|_t|_dSr)r setpartition_namesrrr%r%r&rqeszParquetPartitions.__init__cCs t|jSr)r3r rrr%r%r&__len__iszParquetPartitions.__len__cCs |j|Sr)r )rorr%r%r& __getitem__lszParquetPartitions.__getitem__cCs*t|tstd|j|jko(|j|jkS)Nz0`other` must be an instance of ParquetPartitions)r,rrr rrr%r%r&ros    zParquetPartitions.equalscCs*z ||WStk r$tYSXdSrrrNotImplementedrr%r%r&rvs zParquetPartitions.__eq__cCsV|t|jkrF||jkr&td|t|}|j||j||j||S)ae Record a partition value at a particular level, returning the distinct code for that value at that level. Examples -------- partitions.get_index(1, 'foo', 'a') returns 0 partitions.get_index(1, 'foo', 'b') returns 1 partitions.get_index(1, 'foo', 'c') returns 2 partitions.get_index(1, 'foo', 'a') returns 0 Parameters ---------- level : int The nesting level of the partition we are observing name : str The partition name key : str or int The partition value z1{} was the name of the partition in another level) r3r rr:rOrrVaddr)rolevelrrZpart_setr%r%r&r|s   zParquetPartitions.get_indexc Cs\|\}}|\}}}||krdSt|} |dkrt|tsDtd| j|sPtdtdd|Ddkrptd|ttt|} nt|t st|trtd || |j |j | } |d ks|d kr| |kS|d kr| |kS|d kr| |kS|dkr| |kS|dkr| |kS|dkr&| |kS|dkr8| |kS|dkrJ| |kStd|ddS)NT>rKrLz'%s' object is not a collectionz+Cannot use empty collection as filter valuecSsh|] }t|qSr%)r)r5itemr%r%r& sz=ParquetPartitions.filter_accepts_partition..rzz8All elements of the collection '%s' must be of same typez-Op '%s' not supported with a collection valuerDrErFrGrHrIrJrKrLz+'%s' is not a valid operator in predicates.) rr,rrrr:r3nextiterr/r r Zas_py) ropart_keyfilterr%Zp_columnZ p_value_indexZf_columnrBZf_valueZf_typeZp_valuer%r%r&filter_accepts_partitionsZ         z*ParquetPartitions.filter_accepts_partitionN) rrrrqr r!rrrr,r%r%r%r&rcs!rc@s>eZdZdddZddZd d Zd d Zd dZddZdS)ParquetManifestN/hiverzcCst||\}}||_||_||_t||_||_t|_g|_ ||_ t j |d|_ d|_d|_|d|jg|j jddd|jdkr|j|_|j dS)N) max_workersrcSs|jSr)r$piecer%r%r&z*ParquetManifest.__init__..)r)r*rrpathseprdirpathpartition_schemerr pieces_metadata_nthreadsrThreadPoolExecutor _thread_poolcommon_metadata_path metadata_path _visit_levelsortshutdown)ror6rrr5r7metadata_nthreadsr%r%r&rqs&  zParquetManifest.__init__c sj}t|\}}}g}|D]P} j| f} | drH| _q"| drZ| _q"| rhq"q"| | q"fdd|D} | | t |dkrt | dkrt d n(t | dkr܈|| |n ||dS)N_common_metadata _metadatacs$g|]}t|sj|fqSr%)_is_private_directoryr5r}r base_pathror%r&rRsz0ParquetManifest._visit_level..rz,Found files in an intermediate directory: {})rr(walkr5r}endswithr<r=_should_silently_excluderVr?r3r:rO_visit_directories _push_pieces) ror%rF part_keysfsr directoriesfilesZfiltered_filesr$ full_pathZfiltered_directoriesr%rEr&r>s0      zParquetManifest._visit_levelcCs0|dp.|dp.|dp.|dp.|tkS)Nz.crcz _$folder$r{r)rH startswithEXCLUDED_PARQUET_PATHS)ro file_namer%r%r&rI s z(ParquetManifest._should_silently_excludec Csg}|D]~}t||j\}}t|\}} |j||| } ||| fg} ||jkrt|j|j|d|| } | | q||d|| q|rt |dSNrz) _path_splitr5_parse_hive_partitionr rr9r;submitr>rVrwait) ror%rNrLZ futures_listr$headtailrrrZ dir_part_keysfuturer%r%r&rJs     z"ParquetManifest._visit_directoriescCs&|jdkrt|Std|jdS)Nr/zpartition schema: {})r7rVr=rO)rodirnamer%r%r&_parse_partition(s  z ParquetManifest._parse_partitioncs jfdd|DdS)Ncsg|]}tj|jdqS))rr)rrrr5r$rLror%r&rR0sz0ParquetManifest._push_pieces..)r8r)rorOrLr%r_r&rK/szParquetManifest._push_pieces)NNr.r/rz) rrrrqr>rIrJr]rKr%r%r%r&r-s !r-cCs"d|krtd||ddS)NrDz3Directory name did not appear to be a partition: {}rz)r:rOsplit)valuer%r%r&rV7s rVcCs,tj|\}}|ds$|do*d|kS)Nrr{rD)osr$r`rQ)rrrZr%r%r&rD>srDcCs:||d}|d|||d}}||}||fSrT)rfindrstrip)r$seprrYrZr%r%r&rUCs rUZ_SUCCESSc@seZdZdZdS)_ParquetDatasetMetadata)rMr`r_r^raN)rrr __slots__r%r%r%r&rfMsrfcCsD|jdk r(t|jtjs(|jj|dd}t|||j|j|j|j dS)Nrr)r]r`r_r^ra) rMr,r(rrlr\r`r_r^ra)rTr$metar%r%r&_open_dataset_fileRs  riz['{}' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.{}aread_dictionary : list, default None List of names or column paths (for nested types) to read directly as DictionaryArray. Only supported for BYTE_ARRAY storage. To read a flat column as dictionary-encoded pass the column name. For nested types, you must pass the full column "path", which could be something like level1.level2.list.item. Refer to the Parquet file's schema to obtain the paths. memory_map : bool, default False If the source is a file path, use a memory map to read file, which can improve performance in some environments. buffer_size : int, default 0 If positive, perform read buffering when deserializing individual column chunks. Otherwise IO calls are unbuffered. partitioning : pyarrow.dataset.Partitioning or str or list of str, default "hive" The partitioning scheme for a partitioned dataset. The default of "hive" assumes directory names with key=value pairs like "/year=2009/month=11". In addition, a scheme like "/2009/11" is also supported, in which case you need to specify the field names or a full schema. See the ``pyarrow.dataset.partitioning()`` function for more details.aGenerate an example PyArrow Table and write it to a partitioned dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_name', ... partition_cols=['year'], ... use_legacy_dataset=False) create a ParquetDataset object from the dataset source: >>> dataset = pq.ParquetDataset('dataset_name/', use_legacy_dataset=False) and read the data: >>> dataset.read().to_pandas() n_legs animal year 0 5 Brittle stars 2019 1 2 Flamingo 2020 2 4 Dog 2021 3 100 Centipede 2021 4 2 Parrot 2022 5 4 Horse 2022 create a ParquetDataset object with filter: >>> dataset = pq.ParquetDataset('dataset_name/', use_legacy_dataset=False, ... filters=[('n_legs','=',4)]) >>> dataset.read().to_pandas() n_legs animal year 0 4 Dog 2021 1 4 Horse 2022 c@s@eZdZdeeeZd;ddZdParquetDataseta" Encapsulates details of reading a complete Parquet dataset possibly consisting of multiple files and partitions in subdirectories. Parameters ---------- path_or_paths : str or List[str] A directory name, single file name, or list of file names. filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise it will be parsed as an URI to determine the filesystem. schema : pyarrow.parquet.Schema Use schema obtained elsewhere to validate file schemas. Alternative to metadata parameter. metadata : pyarrow.parquet.FileMetaData Use metadata obtained elsewhere to validate file schemas. split_row_groups : bool, default False Divide files into pieces for each row group in the file. validate_schema : bool, default True Check that individual file schemas are all the same / compatible. filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], default None Rows which do not match the filter predicate will be removed from scanned data. Partition keys embedded in a nested directory structure will be exploited to avoid loading files at all if they contain no matching rows. If `use_legacy_dataset` is True, filters can only reference partition keys and only a hive-style directory structure is supported. When setting `use_legacy_dataset` to False, also within-file level filtering and different partitioning schemes are supported. {1} metadata_nthreads : int, default 1 How many threads to allow the thread pool which is used to read the dataset metadata. Increasing this is helpful to read partitioned datasets. {0} use_legacy_dataset : bool, default False Set to False to enable the new code path (using the new Arrow Dataset API). Among other things, this allows to pass `filters` for all columns and not only the partition keys, enables different partitioning schemes, etc. pre_buffer : bool, default True Coalesce and issue file reads in parallel to improve performance on high-latency filesystems (e.g. S3). If True, Arrow will use a background I/O thread pool. This option is only supported for use_legacy_dataset=False. If using a filesystem layer that itself performs readahead (e.g. fsspec's S3FS), disable readahead for best results. coerce_int96_timestamp_unit : str, default None Cast timestamps that are stored in INT96 format to a particular resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' and therefore INT96 timestamps will be inferred as timestamps in nanoseconds. thrift_string_size_limit : int, default None If not None, override the maximum total string size allocated when decoding Thrift structures. The default limit should be sufficient for most Parquet files. thrift_container_size_limit : int, default None If not None, override the maximum total size of containers allocated when decoding Thrift structures. The default limit should be sufficient for most Parquet files. Examples -------- {2} NFTrr/cCstd}| dkr&t|tjr"d} d}nd} | sRt|||| | | | |||||||||dStjd|tddt|}|S) NrTz The legacy behaviour was still chosen because a deprecated 'pyarrow.filesystem' filesystem was specified (use the filesystems from pyarrow.fs instead).F)rr> partitioningr_r`rarbrcrr]split_row_groupsvalidate_schemarArerfzPassing 'use_legacy_dataset=True' to get the legacy behaviour is deprecated as of pyarrow 11.0.0, and the legacy implementation will be removed in a future version.rr) r,r(r_ParquetDatasetV2rrrobjectr)cls path_or_pathsrrr]rlrmr>rAr_r`rarkuse_legacy_datasetrbrcrerfZ extra_msgror%r%r&rsF   zParquetDataset.__new__c Cs| dkrtd|dk r*tjdtddnd}t|_|}t|trL|d}t||\|j_ }t|trzdd |D|_ n t ||_ | |j_ | |j_ | |j_t||j|tt|jd \|_|_|_|_|jdk r|j|j}t|| d |j_W5QRXnd|j_|dk rtjd tdd|dkr^|jdk r^|j|j}t|| d |_W5QRXn||_|dk r~tjd tdd||_||_|rtd|dk rt|drtdt |}|!||r|"dS)Nr/zVOnly "hive" for hive-like partitioning is supported when using use_legacy_dataset=TruezSpecifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future versionrrrzrcSsg|] }t|qSr%)r'r^r%r%r&rR6sz+ParquetDataset.__init__..)rAr)r`zdSpecifying the 'metadata' argument with 'use_legacy_dataset=True' is deprecated as of pyarrow 8.0.0.zSpecifying the 'schema' argument with 'use_legacy_dataset=True' is deprecated as of pyarrow 8.0.0. You can still specify it in combination with 'use_legacy_dataet=False', but in that case you need to specify a pyarrow.Schema instead of a ParquetSchema.z$split_row_groups not yet implementedcastz6Expressions as filter not supported for legacy dataset)#r:rrrrf _ds_metadatar,r;r*rMr~r'r_r`ra_make_manifest_fsrri_pieces _partitions_common_metadata_path_metadata_pathrl read_metadatar^rC_schemarlr=hasattrrrC_filtervalidate_schemas)rorqrrr]rlrmr>rAr_r`rarkrrrbrcrerfZa_pathrr6r%r%r&rqs          zParquetDataset.__init__cCsdtddfS)Nr%T)rr)rrrr%r%r&__getnewargs_ex__tsz ParquetDataset.__getnewargs_ex__cCsvt|tstd|jj|jjkr&dSdD]}t||t||kr*dSq*dD]"}t|j|t|j|krNdSqNdS)N-`other` must be an instance of ParquetDatasetF) r~rwrxryrzrBrCr|rl)r`raT)r,rjrrv __class__rhrt)rorpropr%r%r&rzs   zParquetDataset.equalscCs*z ||WStk r$tYSXdSrr"rr%r%r&rs zParquetDataset.__eq__cCs|jdkr>|jdkr>|jdk r*|jj|_qR|jdj|_n|jdkrR|jj|_|j}|jdk r|jjD]&}| |dkrn| |}| |}qn|jD]6}|}|j}|j |ddst d |||qdS)NrFrz-Schema in {!s} was different. {!s} vs {!s})rCr|rBrrwrto_arrow_schemarxrget_field_indexremoverr:rO)roZdataset_schemaZpartition_nameZ field_idxr2Z file_metadataZ file_schemar%r%r&rs*           zParquetDataset.validate_schemasc Csng}|jD]"}|j|||j|d}||q t|}|rj|}|jjpNi} |rjd| krj| d|i}|S)a Read multiple Parquet files as a single pyarrow.Table. Parameters ---------- columns : List[str] Names of columns to read from the file. use_threads : bool, default True Perform multi-threaded column reads use_pandas_metadata : bool, default False Passed through to each dataset piece. Returns ------- pyarrow.Table Content of the file as a table (of columns). Examples -------- Generate an example dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_name_read', ... partition_cols=['year'], ... use_legacy_dataset=False) >>> dataset = pq.ParquetDataset('dataset_name_read/', ... use_legacy_dataset=False) Read multiple Parquet files as a single pyarrow.Table: >>> dataset.read(columns=["n_legs"]) pyarrow.Table n_legs: int64 ---- n_legs: [[5],[2],[4,100],[2,4]] )rrr rr) rwrrxrVrZ concat_tables_get_common_pandas_metadatarr]replace_schema_metadata) rorrrZtablesr2rZall_datar^Zcurrent_metadatar%r%r&rs$*     zParquetDataset.readcKs|jfddi|S)a Read dataset including pandas metadata, if any. Other arguments passed through to ParquetDataset.read, see docstring for further details. Parameters ---------- **kwargs : optional All additional options to pass to the reader. Returns ------- pyarrow.Table Content of the file as a table (of columns). Examples -------- Generate an example PyArrow Table and write it to a partitioned dataset: >>> import pyarrow as pa >>> import pandas as pd >>> df = pd.DataFrame({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> table = pa.Table.from_pandas(df) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'table.parquet') >>> dataset = pq.ParquetDataset('table.parquet', ... use_legacy_dataset=False) Read dataset including pandas metadata: >>> dataset.read_pandas(columns=["n_legs"]) pyarrow.Table n_legs: int64 ---- n_legs: [[2,2,4,4,5,100]] Select pandas metadata: >>> dataset.read_pandas(columns=["n_legs"]).schema.pandas_metadata {'index_columns': [{'kind': 'range', 'name': None, 'start': 0, ...} rTrrorxr%r%r& read_pandass-zParquetDataset.read_pandascCs"|jdkrdS|jj}|ddS)Nr)rBr]get)rorr%r%r&r!s z*ParquetDataset._get_common_pandas_metadatacs<|jjfddfddfdd|jD|_dS)Ncstfddt|jDS)Nc3s|]\}}||VqdSrr%)r5r%r*)accepts_filterr+r%r&r7,szEParquetDataset._filter..one_filter_accepts..)r<r|r)r2r+)r)r+r&one_filter_accepts+sz2ParquetDataset._filter..one_filter_acceptscstfddDS)Nc3s&|]}tfdd|DVqdS)c3s|]}|VqdSrr%r4rr2r%r&r70szOParquetDataset._filter..all_filters_accept...N)r<)r5r@rr%r&r70szEParquetDataset._filter..all_filters_accept..)r9r1)r>rr1r&all_filters_accept/sz2ParquetDataset._filter..all_filters_acceptcsg|]}|r|qSr%r%)r5p)rr%r&rR3sz*ParquetDataset._filter..)rxr,rw)ror>r%)rrr>rr&r~(s zParquetDataset._filtercCstjtddtdd|jS) DEPRECATED ParquetDataset.piecesz{ Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.rr)rr _DEPR_MSGrOrrwrrr%r%r&r85srcCstjtddtdd|jS)rParquetDataset.partitionsz~ Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.rr)rrrrOrrxrrr%r%r&r CsrcCstjtddtdd|jS)NParquetDataset.schemaz Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.schema' attribute instead (which will return an Arrow schema instead of a Parquet schema).rr)rrrrOrr|rrr%r%r&rQsrcCs tjtddtdd|jjS)rParquetDataset.memory_maprrr)rrrrOrrtr`rrr%r%r&r`]s  rcCs tjtddtdd|jjS)rParquetDataset.read_dictionaryrrr)rrrrOrrtr_rrr%r%r&r_gs  rcCs tjtddtdd|jjS)rParquetDataset.buffer_sizerrr)rrrrOrrtrarrr%r%r&raqs  rz_ds_metadata.fscCs tjtddtdd|jjS)rParquetDataset.fsz| Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.rr)rrrrOrrtrMrrr%r%r&rMsrcCstjtddtdd|jS)rParquetDataset.metadatarrr)rrrrOrrCrrr%r%r&r]s  rcCstjtddtdd|jS)rParquetDataset.metadata_pathrrr)rrrrOrrzrrr%r%r&r=s  rcCstjtddtdd|jS)r#ParquetDataset.common_metadata_pathrrr)rrrrOrryrrr%r%r&r<s  rz_ds_metadata.common_metadatacCs tjtddtdd|jjS)rParquetDataset.common_metadatarrr)rrrrOrrtr^rrr%r%r&r^s  rcCs tddS)aL A list of the Dataset source fragments or pieces with absolute file paths. To use this property set 'use_legacy_dataset=False' while constructing ParquetDataset object. Examples -------- Generate an example dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_name_fragments', ... partition_cols=['year'], ... use_legacy_dataset=False) >>> dataset = pq.ParquetDataset('dataset_name_fragments/', ... use_legacy_dataset=False) List the fragments: >>> dataset.fragments [>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_name_files', ... partition_cols=['year'], ... use_legacy_dataset=False) >>> dataset = pq.ParquetDataset('dataset_name_files/', ... use_legacy_dataset=False) List the files: >>> dataset.files ['dataset_name_files/year=2019/...-0.parquet', ... rNrrrr%r%r&rOszParquetDataset.filescCs tddS)z The filesystem type of the Dataset source. To use this property set 'use_legacy_dataset=False' while constructing ParquetDataset object. rNrrrr%r%r&rszParquetDataset.filesystemcCs tddS)z The partitioning of the Dataset source, if discovered. To use this property set 'use_legacy_dataset=False' while constructing ParquetDataset object. rNrrrr%r%r&rk szParquetDataset.partitioning)NNNNFTNNNFrr/NTNNN)NNNFTNNNFrr/NTNNN)NTF)&rrrrO_read_docstring_common_DNF_filter_doc_parquet_dataset_examplerrrqrrrrrrrr~rr8r rr`r_rarW attrgetterrvrMr]r=r<rBr^rrOrrkr%r%r%r&rjs@B 1 [ A/               rjr.rzc Csd}d}d}t|tr*t|dkr*|d}t|rp||rpt|||t|dd|d}|j}|j}|j } |j }nbt|ts|g}t|dkrt dg} |D]4} | | st d| tj| |d} | | q| |||fS) Nrzrr5r.)rrr5rAz Must pass at least one file pathzPassed non-file path: {})r)r,r;r3risdirr-rhr<r=r8r r:isfileOSErrorrOrrrV) rqrMr5rArr r<r=manifestr8r$r2r%r%r&ru s>     rucCst|tpt|tjSr)r,rr()rMr%r%r&_is_local_file_system7 s rc@seZdZdZdddddddddddddd ddZd d Zd d Zed dZd ddZ ddZ ddZ eddZ eddZ eddZeddZeddZdS)!rna ParquetDataset shim using the Dataset API under the hood. Examples -------- Generate an example PyArrow Table and write it to a partitioned dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_v2', ... partition_cols=['year'], ... use_legacy_dataset=False) create a ParquetDataset object from the dataset source: >>> dataset = pq.ParquetDataset('dataset_v2/', use_legacy_dataset=False) and read the data: >>> dataset.read().to_pandas() n_legs animal year 0 5 Brittle stars 2019 1 2 Flamingo 2020 2 4 Dog 2021 3 100 Centipede 2021 4 2 Parrot 2022 5 4 Horse 2022 create a ParquetDataset object with filter: >>> dataset = pq.ParquetDataset('dataset_v2/', ... filters=[('n_legs','=',4)], ... use_legacy_dataset=False) >>> dataset.read().to_pandas() n_legs animal year 0 4 Dog 2021 1 4 Horse 2022 Nr/FT) r>rkr_rar`ignore_prefixesrbrcrrdrerfc Ksddlm}dD]*\}}||kr|||k rtd|q| | | |d}|r\|jd|d|dk rp|j|d| dk r|j| dd|_|dk rt||_|dk rt||d }n|dkr|rt|d }t |d r|dk rt |st d t |d}d|_ t|tst|rt|}|dkrZzt|\}}Wn tk rXt|d }YnX||}|jrp|}|j tjkr||_ n|}|jf|}|dk r|||}|j|g| p|j||jd |_dS|d kr|jjdd}|j||| |||d|_dS)Nr))r]N)rlF)rmT)rANz;Keyword '{0}' is not yet supported with the new Dataset API)rbrcrerfT)Zuse_buffered_streamra)Zdictionary_columns)rd)Zuse_mmap __fspath__zQPath-like objects with __fspath__ must only be used with local file systems, not )rrOrr/)Zinfer_dictionary)rrrOrkr) rSrTr:rOupdate_filter_expressionrZrrr}rrr _base_dirr,r;rrrZfrom_uri get_file_infois_filerZ DirectoryParquetFileFormatZ make_fragmentZFileSystemDatasetZphysical_schemar_datasetZHivePartitioningZdiscover)rorqrr>rkr_rar`rrbrcrrdrerfrxrPkeyworddefaultZ read_optionsZ single_filefinfoparquet_formatfragmentr%r%r&rqi s                  z_ParquetDatasetV2.__init__cCsXt|trtdt|ts$td|j|jkoV|jj|jjkoV|j|jkoV|j|jkS)NzY`other` must be an instance of ParquetDataset constructed with `use_legacy_dataset=False`r) r,rjrrnrrrOrrOrr%r%r&r s     z_ParquetDatasetV2.equalscCs*z ||WStk r$tYSXdSrr"rr%r%r&r s z_ParquetDatasetV2.__eq__cCs|jjS)a Schema of the Dataset. Examples -------- Generate an example dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_v2_schema', ... partition_cols=['year'], ... use_legacy_dataset=False) >>> dataset = pq.ParquetDataset('dataset_v2_schema/', ... use_legacy_dataset=False) Read the schema: >>> dataset.schema n_legs: int64 animal: string year: dictionary )rrrrr%r%r&r sz_ParquetDatasetV2.schemac Cs|jjp i}|r(d|kr(|}|r(|}|dk rn|rn|rnd|krnddt|D}t|tt|t|}|jj||j|d}|r|rd|kr|jjpi}| d|di| |}|S)aR Read (multiple) Parquet files as a single pyarrow.Table. Parameters ---------- columns : List[str] Names of columns to read from the dataset. The partition fields are not automatically included (in contrast to when setting ``use_legacy_dataset=True``). use_threads : bool, default True Perform multi-threaded column reads. use_pandas_metadata : bool, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded. Returns ------- pyarrow.Table Content of the file as a table (of columns). Examples -------- Generate an example dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_v2_read', ... partition_cols=['year'], ... use_legacy_dataset=False) >>> dataset = pq.ParquetDataset('dataset_v2_read/', ... use_legacy_dataset=False) Read the dataset: >>> dataset.read(columns=["n_legs"]) pyarrow.Table n_legs: int64 ---- n_legs: [[5],[2],[4,100],[2,4]] rNcSsg|]}t|ts|qSr%)r,rr5rAr%r%r&rR> s z*_ParquetDatasetV2.read..)rr+r) rr]rrr;rrZto_tablerrr) rorrrr]r^rrZ new_metadatar%r%r&r s0/      z_ParquetDatasetV2.readcCsf|js dSd}dD]N}tjt|j|}|j|}|jrt||jd}|j }|rd|krqbq|S)N)rBrCrr) rrbr$r}r/rrrr{r])ror]rr=rZpq_metar%r%r&rU s  z-_ParquetDatasetV2._get_common_pandas_metadatacKs|jfddi|S)a Read dataset including pandas metadata, if any. Other arguments passed through to ParquetDataset.read, see docstring for further details. Examples -------- Generate an example parquet file: >>> import pyarrow as pa >>> import pandas as pd >>> df = pd.DataFrame({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> table = pa.Table.from_pandas(df) >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'table_V2.parquet') >>> dataset = pq.ParquetDataset('table_V2.parquet', ... use_legacy_dataset=False) Read the dataset with pandas metadata: >>> dataset.read_pandas(columns=["n_legs"]) pyarrow.Table n_legs: int64 ---- n_legs: [[2,2,4,4,5,100]] >>> dataset.read_pandas(columns=["n_legs"]).schema.pandas_metadata {'index_columns': [{'kind': 'range', 'name': None, 'start': 0, ...} rTrrr%r%r&rg s z_ParquetDatasetV2.read_pandascCs&tjtddtddt|jS)Nrz' Use the '.fragments' attribute insteadrr)rrrrOrr;r get_fragmentsrrr%r%r&r8 sz_ParquetDatasetV2.piecescCst|jS)a A list of the Dataset source fragments or pieces with absolute file paths. Examples -------- Generate an example dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_v2_fragments', ... partition_cols=['year'], ... use_legacy_dataset=False) >>> dataset = pq.ParquetDataset('dataset_v2_fragments/', ... use_legacy_dataset=False) List the fragments: >>> dataset.fragments [>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_v2_files', ... partition_cols=['year'], ... use_legacy_dataset=False) >>> dataset = pq.ParquetDataset('dataset_v2_files/', ... use_legacy_dataset=False) List the files: >>> dataset.files ['dataset_v2_files/year=2019/...-0.parquet', ... )rrOrrr%r%r&rO sz_ParquetDatasetV2.filescCs|jjS)z< The filesystem type of the Dataset source. )rrrrr%r%r&r sz_ParquetDatasetV2.filesystemcCs|jjS)zH The partitioning of the Dataset source, if discovered. )rrkrrr%r%r&rk sz_ParquetDatasetV2.partitioning)N)NTF)rrrrrqrrrrrrrr8rrOrrkr%r%r%r&rn= s>+ f  S"    rna {0} Parameters ---------- source : str, pyarrow.NativeFile, or file-like object If a string passed, can be a single file name or directory name. For file-like objects, only read a single file. Use pyarrow.BufferReader to read a file contained in a bytes or buffer-like object. columns : list If not None, only these columns will be read from the file. A column name may be a prefix of a nested field, e.g. 'a' will select 'a.b', 'a.c', and 'a.d.e'. If empty, no columns will be read. Note that the table will still have the correct num_rows set despite having no columns. use_threads : bool, default True Perform multi-threaded column reads. metadata : FileMetaData If separately computed schema : Schema, optional Optionally provide the Schema for the parquet dataset, in which case it will not be inferred from the source. {1} filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise it will be parsed as an URI to determine the filesystem. filters : pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]], default None Rows which do not match the filter predicate will be removed from scanned data. Partition keys embedded in a nested directory structure will be exploited to avoid loading files at all if they contain no matching rows. If `use_legacy_dataset` is True, filters can only reference partition keys and only a hive-style directory structure is supported. When setting `use_legacy_dataset` to False, also within-file level filtering and different partitioning schemes are supported. {3} use_legacy_dataset : bool, default False By default, `read_table` uses the new Arrow Datasets API since pyarrow 1.0.0. Among other things, this allows to pass `filters` for all columns and not only the partition keys, enables different partitioning schemes, etc. Set to True to use the legacy behaviour (this option is deprecated, and the legacy implementation will be removed in a future version). ignore_prefixes : list, optional Files matching any of these prefixes will be ignored by the discovery process if use_legacy_dataset=False. This is matched to the basename of a path. By default this is ['.', '_']. Note that discovery happens only if a directory is passed as source. pre_buffer : bool, default True Coalesce and issue file reads in parallel to improve performance on high-latency filesystems (e.g. S3). If True, Arrow will use a background I/O thread pool. This option is only supported for use_legacy_dataset=False. If using a filesystem layer that itself performs readahead (e.g. fsspec's S3FS), disable readahead for best results. coerce_int96_timestamp_unit : str, default None Cast timestamps that are stored in INT96 format to a particular resolution (e.g. 'ms'). Setting to None is equivalent to 'ns' and therefore INT96 timestamps will be inferred as timestamps in nanoseconds. decryption_properties : FileDecryptionProperties or None File-level decryption properties. The decryption properties can be created using ``CryptoFactory.file_decryption_properties()``. thrift_string_size_limit : int, default None If not None, override the maximum total string size allocated when decoding Thrift structures. The default limit should be sufficient for most Parquet files. thrift_container_size_limit : int, default None If not None, override the maximum total size of containers allocated when decoding Thrift structures. The default limit should be sufficient for most Parquet files. Returns ------- {2} {4} ad Examples -------- Generate an example PyArrow Table and write it to a partitioned dataset: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_name_2', ... partition_cols=['year']) Read the data: >>> pq.read_table('dataset_name_2').to_pandas() n_legs animal year 0 5 Brittle stars 2019 1 2 Flamingo 2020 2 4 Dog 2021 3 100 Centipede 2021 4 2 Parrot 2022 5 4 Horse 2022 Read only a subset of columns: >>> pq.read_table('dataset_name_2', columns=["n_legs", "animal"]) pyarrow.Table n_legs: int64 animal: string ---- n_legs: [[5],[2],[4,100],[2,4]] animal: [["Brittle stars"],["Flamingo"],["Dog","Centipede"],["Parrot","Horse"]] Read a subset of columns and read one column as DictionaryArray: >>> pq.read_table('dataset_name_2', columns=["n_legs", "animal"], ... read_dictionary=["animal"]) pyarrow.Table n_legs: int64 animal: dictionary ---- n_legs: [[5],[2],[4,100],[2,4]] animal: [ -- dictionary: ["Brittle stars"] -- indices: [0], -- dictionary: ["Flamingo"] -- indices: [0], -- dictionary: ["Dog","Centipede"] -- indices: [0,1], -- dictionary: ["Parrot","Horse"] -- indices: [0,1]] Read the table with filter: >>> pq.read_table('dataset_name_2', columns=["n_legs", "animal"], ... filters=[('n_legs','<',4)]).to_pandas() n_legs animal 0 2 Flamingo 1 2 Parrot Read data from a single Parquet file: >>> pq.write_table(table, 'example.parquet') >>> pq.read_table('dataset_name_2').to_pandas() n_legs animal year 0 5 Brittle stars 2019 1 2 Flamingo 2020 2 4 Dog 2021 3 100 Centipede 2021 4 2 Parrot 2022 5 4 Horse 2022 Fr/)rrr]rrr_r`rarkrr>rrrrbrcrdrerfcCsl| s|dk rtdz&t||| | |||| | ||||d }Wntk r| dk rZtd| dkrjtd|dk rztdt|| \} }| dk r| |}t||||||||||d }YnX|j|||dStjd t d d | dk rtd |dk rtd t |rFt 0t ddt t |||||| | | |dd }W5QRXnt|||||||d}|j|||dS)NzThe 'metadata' keyword is no longer supported with the new datasets-based implementation. Specify 'use_legacy_dataset=True' to temporarily recover the old behaviour.) rrrkr`r_rar>rrbrcrerfzWthe 'filters' keyword is not supported when the pyarrow.dataset module is not availabler/z\the 'partitioning' keyword is not supported when the pyarrow.dataset module is not availablezWthe 'schema' argument is not supported when the pyarrow.dataset module is not available) r]r_r`rarbrcrdrerfrPassing 'use_legacy_dataset=True' to get the legacy behaviour is deprecated as of pyarrow 8.0.0, and the legacy implementation will be removed in a future version.rrzMThe 'ignore_prefixes' keyword is only supported when use_legacy_dataset=FalsezEThe 'schema' argument is only supported when use_legacy_dataset=FalseignorezPassing 'use_legacy_datasetT) r]r`r_rarr>rkrcrr)r]r_r`rarcrd)r:rn ImportErrorrrjr\rrrrrcatch_warningsfilterwarningsrj)rprrr]rrr_r`rarkrr>rrrrbrcrdrerfrTr$pfr%r%r& read_tablez s      rz}Read a Table from Parquet format Note: starting with pyarrow 1.0, the default for `use_legacy_dataset` is switched to False. zuse_pandas_metadata : bool, default False If True and file has custom pandas schema metadata, ensure that index columns are also loaded.z=pyarrow.Table Content of the file as a table (of columns)cKst|f|dd|S)NT)rr)r)rprrxr%r%r&r srzcRead a Table from Parquet format, also reading DataFrame index values if known in the file metadataz6**kwargs additional options for :func:`read_table`zgpyarrow.Table Content of the file as a Table of Columns, including DataFrame indexes as columnsrrrrcKs|d|}|}z\t||jf| || |||| | ||| |||||||||d|}|j||dW5QRXWnHtk rt|rztt|Wntj k rYnXYnXdS)N chunk_size)rrrrrcoerce_timestampsdata_page_sizeallow_truncated_timestampsrrrrrrrrrrrrr) rrrr Exceptionrrbrrerror)rrrrrrrrrrrrrrrrrrrrrrrrxZ use_int96rr%r%r&r sN raGenerate an example PyArrow Table: >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) and write the Table into Parquet file: >>> import pyarrow.parquet as pq >>> pq.write_table(table, 'example.parquet') Defining row group size for the Parquet file: >>> pq.write_table(table, 'example.parquet', row_group_size=3) Defining row group compression (default is Snappy): >>> pq.write_table(table, 'example.parquet', compression='none') Defining row group compression and encoding per-column: >>> pq.write_table(table, 'example.parquet', ... compression={'n_legs': 'snappy', 'animal': 'gzip'}, ... use_dictionary=['n_legs', 'animal']) Defining column encoding per-column: >>> pq.write_table(table, 'example.parquet', ... column_encoding={'animal':'PLAIN'}, ... use_dictionary=False) ap Write a Table to Parquet format. Parameters ---------- table : pyarrow.Table where : string or pyarrow.NativeFile row_group_size : int Maximum number of rows in each written row group. If None, the row group size will be the minimum of the Table size and 1024 * 1024. {} **kwargs : optional Additional options for ParquetWriter Examples -------- {} cCsH|rD||sDz||Wn"tk rB||s>tYnXdSr)Z _isfilestoreexistsmkdirrr)rMr$r%r%r&_mkdir_if_not_existsw s rc & s|dkr|rd}nd}d} d}|r&| n|}|dk rL|dk rLt|dd||dk rn|dk rnt|dd || d ddk r| dk rt|d d ||sd dlm}t}t|jj D]}|| kr| |||<q| d | dd|d<d}dk rfdd} |dk r(t|d| }|j f| }|dk rNt |}|rn| |j}|j|dd}|dkrtd}| dkrd} |j||f|||||| | || d |dS|rtjdtddd}|dk rt|d|dk rt|d| dk rt|d| dk r6t|d | dk rNt|d|dk rft|d|dk rtjtddtddt||\}}t|||dk r4t|d kr4|fd d!|D}j|d"d#}j|}t|d krtd$|j}|jjD] }||kr|||}qt|d%krJ|d }|j|dd&D]\}}t |t!sr|f}d'"d(d!t#||D} t$j%j&||dd)}!t|d'"|| g|r||}"n td*}"d'"| |"g}#d'"||#g}$|'|$d+}%t(|!|%fd i| W5QRXdk rXd,)|#qXnn|rD|d}"n td*}"d'"||"g}$|'|$d+}%t(||%fd i| W5QRXdk rd,)|"dS)-aWrapper around dataset.write_dataset (when use_legacy_dataset=False) or parquet.write_table (when use_legacy_dataset=True) for writing a Table to Parquet format by partitions. For each combination of partition columns and values, a subdirectories are created in the following manner: root_dir/ group1=value1 group2=value1 .parquet group2=value2 .parquet group1=valueN group2=value1 .parquet group2=valueN .parquet Parameters ---------- table : pyarrow.Table root_path : str, pathlib.Path The root directory of the dataset partition_cols : list, Column names by which to partition the dataset. Columns are partitioned in the order they are given partition_filename_cb : callable, A callback function that takes the partition key(s) as an argument and allow you to override the partition filename. If nothing is passed, the filename will consist of a uuid. This option is only supported for use_legacy_dataset=True. When use_legacy_dataset=None and this option is specified, use_legacy_datase will be set to True. filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise it will be parsed as an URI to determine the filesystem. use_legacy_dataset : bool Default is False. Set to True to use the the legacy behaviour (this option is deprecated, and the legacy implementation will be removed in a future version). The legacy implementation still supports the `partition_filename_cb` keyword but is less efficient when using partition columns. schema : Schema, optional This option is only supported for use_legacy_dataset=False. partitioning : Partitioning or list[str], optional The partitioning scheme specified with the ``pyarrow.dataset.partitioning()`` function or a list of field names. When providing a list of field names, you can use ``partitioning_flavor`` to drive which partitioning type should be used. This option is only supported for use_legacy_dataset=False. basename_template : str, optional A template string used to generate basenames of written data files. The token '{i}' will be replaced with an automatically incremented integer. If not specified, it defaults to "guid-{i}.parquet". This option is only supported for use_legacy_dataset=False. use_threads : bool, default True Write files in parallel. If enabled, then maximum parallelism will be used determined by the number of available CPU cores. This option is only supported for use_legacy_dataset=False. file_visitor : function If set, this function will be called with a WrittenFile instance for each file created during the call. This object will have both a path attribute and a metadata attribute. The path attribute will be a string containing the path to the created file. The metadata attribute will be the parquet metadata of the file. This metadata will have the file path attribute set and can be used to build a _metadata file. The metadata attribute will be None if the format is not parquet. Example visitor which simple collects the filenames created:: visited_paths = [] def file_visitor(written_file): visited_paths.append(written_file.path) This option is only supported for use_legacy_dataset=False. existing_data_behavior : 'overwrite_or_ignore' | 'error' | 'delete_matching' Controls how the dataset will handle data that already exists in the destination. The default behaviour is 'overwrite_or_ignore'. 'overwrite_or_ignore' will ignore any existing data and will overwrite files with the same name as an output file. Other existing files will be ignored. This behavior, in combination with a unique basename_template for each write, will allow for an append workflow. 'error' will raise an error if any data exists in the destination. 'delete_matching' is useful when you are writing a partitioned dataset. The first time each partition directory is encountered the entire directory will be deleted. This allows you to overwrite old partitions completely. This option is only supported for use_legacy_dataset=False. **kwargs : dict, When use_legacy_dataset=False, used as additional kwargs for `dataset.write_dataset` function for matching kwargs, and remainder to `ParquetFileFormat.make_write_options`. See the docstring of `write_table` and `dataset.write_dataset` for the available options. When use_legacy_dataset=True, used as additional kwargs for `parquet.write_table` function (See docstring for `write_table` or `ParquetWriter` for more information). Using `metadata_collector` in kwargs allows one to collect the file metadata instances of dataset pieces. The file paths in the ColumnChunkMetaData will be set relative to `root_path`. Examples -------- Generate an example PyArrow Table: >>> import pyarrow as pa >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], ... 'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) and write it to a partitioned dataset: >>> import pyarrow.parquet as pq >>> pq.write_to_dataset(table, root_path='dataset_name_3', ... partition_cols=['year']) >>> pq.ParquetDataset('dataset_name_3', use_legacy_dataset=False).files ['dataset_name_3/year=2019/...-0.parquet', ... Write a single Parquet file into the root folder: >>> pq.write_to_dataset(table, root_path='dataset_name_4') >>> pq.ParquetDataset('dataset_name_4/', use_legacy_dataset=False).files ['dataset_name_4/...-0.parquet'] NTFzVThe '{0}' argument is not supported by use_legacy_dataset={2}. Use only '{1}' instead.zVThe '{1}' argument is not supported by use_legacy_dataset={2}. Use only '{0}' instead.basename_templatepartition_filename_cbrkpartition_colsr file_visitorrrrZmax_rows_per_groupzGThe '{}' argument is not supported with the new dataset implementation.cs|jdSr)rVr])Z written_file)rr%r&rF sz&write_to_dataset..file_visitorr/)rz -{i}.parquetZoverwrite_or_ignore) rrOrrrkrrrexisting_data_behaviorrrrzThe '{}' argument is not supported with the legacy implementation. To use this argument specify 'use_legacy_dataset=False' while constructing the ParquetDataset.rrrz Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the 'basename_template' parameter instead. For usage see `pyarrow.dataset.write_dataset`csg|] }|qSr%r%r)dfr%r&rR sz$write_to_dataset..r)Zaxisz.No data left to save outside partition columnsrz)Zobservedr.cSsg|]\}}dj||dqS)z{colname}={value})Zcolnamerar)r5rr0r%r%r&rR s)rsafez.parquetrr)*r:rOrrSrTrinspect signatureZ write_dataset parametersrZmake_write_optionsrselectrrkrrrrrr(r)rr3Z to_pandasZdroprnamesrrgroupbyr,tupler}ziprrZ from_pandasrlrZ set_file_path)&r root_pathrrrrrrrkrrrrrxZ msg_confl_0Z msg_confl_1Z msg_conflrPZwrite_dataset_kwargsrrrZ write_optionsZ part_schemaZmsg2rMrZdata_dfZ data_colsZ subschemarArZsubgroupsubdirZsubtableoutfile relative_pathrPr6r%)rrr&write_to_dataset s                                   rc Kst||\}}t|dr |}t|||f|}||dk rt||d}t|dr`|||D]}||qd|dk r||} | | W5QRXn | |dS)a& Write metadata-only Parquet file from schema. This can be used with `write_to_dataset` to generate `_common_metadata` and `_metadata` sidecar files. Parameters ---------- schema : pyarrow.Schema where : string or pyarrow.NativeFile metadata_collector : list where to collect metadata information. filesystem : FileSystem, default None If nothing passed, will be inferred from `where` if path-like, else `where` is already a file-like object so no filesystem is needed. **kwargs : dict, Additional kwargs for ParquetWriter class. See docstring for `ParquetWriter` for more information. Examples -------- Generate example data: >>> import pyarrow as pa >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", ... "Brittle stars", "Centipede"]}) Write a dataset and collect metadata information. >>> metadata_collector = [] >>> import pyarrow.parquet as pq >>> pq.write_to_dataset( ... table, 'dataset_metadata', ... metadata_collector=metadata_collector) Write the `_common_metadata` parquet file without row groups statistics. >>> pq.write_metadata( ... table.schema, 'dataset_metadata/_common_metadata') Write the `_metadata` parquet file with row groups statistics. >>> pq.write_metadata( ... table.schema, 'dataset_metadata/_metadata', ... metadata_collector=metadata_collector) seekNr) rr}tellrrur{rZappend_row_groupsrZwrite_metadata_file) rrrrrxZcursor_positionrr]mr6r%r%r&write_metadata s0      rc CsZt||\}}t}|dk r*||}}|"t|||d}|jW5QRSQRXdS)a Read FileMetaData from footer of a single Parquet file. Parameters ---------- where : str (file path) or file-like object memory_map : bool, default False Create memory map when the source is a file path. decryption_properties : FileDecryptionProperties, default None Decryption properties for reading encrypted Parquet files. filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise it will be parsed as an URI to determine the filesystem. Returns ------- metadata : FileMetaData The metadata of the Parquet file Examples -------- >>> import pyarrow as pa >>> import pyarrow.parquet as pq >>> table = pa.table({'n_legs': [4, 5, 100], ... 'animal': ["Dog", "Brittle stars", "Centipede"]}) >>> pq.write_table(table, 'example.parquet') >>> pq.read_metadata('example.parquet') created_by: parquet-cpp-arrow version ... num_columns: 2 num_rows: 3 num_row_groups: 1 format_version: 2.6 serialized_size: ... Nr`rd)rrrjr\r]rr`rdrZfile_ctxr r%r%r&r{ s'r{c Cs^t||\}}t}|dk r*||}}|&t|||d}|jW5QRSQRXdS)a Read effective Arrow schema from Parquet file metadata. Parameters ---------- where : str (file path) or file-like object memory_map : bool, default False Create memory map when the source is a file path. decryption_properties : FileDecryptionProperties, default None Decryption properties for reading encrypted Parquet files. filesystem : FileSystem, default None If nothing passed, will be inferred based on path. Path will try to be found in the local on-disk filesystem otherwise it will be parsed as an URI to determine the filesystem. Returns ------- schema : pyarrow.Schema The schema of the Parquet file Examples -------- >>> import pyarrow as pa >>> import pyarrow.parquet as pq >>> table = pa.table({'n_legs': [4, 5, 100], ... 'animal': ["Dog", "Brittle stars", "Centipede"]}) >>> pq.write_table(table, 'example.parquet') >>> pq.read_schema('example.parquet') n_legs: int64 animal: string Nr)rrrjr\rrrr%r%r& read_schema>s"r)r rrrr rjrr\rr-rr rrrr r r{rrrrrrr[rZ)T)N)r.rzN)N)NrTrTNNFNNNNFNrTNNNTF) NNNNNNNNNN)NN)FNN)FNN)h collectionsr concurrentr contextlibr functoolsrrrrcollections.abcrnumpyrrbrerW urllib.parserrZpyarrowrZ pyarrow.librZpyarrow._parquetrrexcr/r r r r r rrrrrZ pyarrow.fsrrrrrrr(Z pyarrow.utilrrrrr#r'r*r2rCrrZDeprecationWarningr[r\compilerrrrrrrrrrrr-rVrDrUrRrfrirrrrjrurrnZ_read_table_docstringZ_read_table_examplerrOr}rrrZ_write_table_examplerrrr{r__all__r%r%r%r&sP     0    )C c  3; 0Gik (p 'RO n   9" G H 2 .