Builds options for chunked_parquet_writer_options. More...

#include <parquet.hpp>

Public Member Functions
	chunked_parquet_writer_options_builder ()=default
	Default constructor. More...

	chunked_parquet_writer_options_builder (sink_info const &sink)
	Constructor from sink. More...

chunked_parquet_writer_options_builder &	metadata (table_input_metadata metadata)
	Sets metadata to chunked_parquet_writer_options. More...

chunked_parquet_writer_options_builder &	key_value_metadata (std::vector< std::map< std::string, std::string >> metadata)
	Sets Key-Value footer metadata in parquet_writer_options. More...

chunked_parquet_writer_options_builder &	stats_level (statistics_freq sf)
	Sets the level of statistics in chunked_parquet_writer_options. More...

chunked_parquet_writer_options_builder &	compression (compression_type compression)
	Sets compression type to chunked_parquet_writer_options. More...

chunked_parquet_writer_options_builder &	int96_timestamps (bool enabled)
	Set to true if timestamps should be written as int96 types instead of int64 types. Even though int96 is deprecated and is not an internal type for cudf, it needs to be written for backwards compatibility reasons. More...

chunked_parquet_writer_options_builder &	utc_timestamps (bool enabled)
	Set to true if timestamps are to be written as UTC. More...

chunked_parquet_writer_options_builder &	write_v2_headers (bool enabled)
	Set to true if V2 page headers are to be written. More...

chunked_parquet_writer_options_builder &	row_group_size_bytes (size_t val)
	Sets the maximum row group size, in bytes. More...

chunked_parquet_writer_options_builder &	row_group_size_rows (size_type val)
	Sets the maximum number of rows in output row groups. More...

chunked_parquet_writer_options_builder &	max_page_size_bytes (size_t val)
	Sets the maximum uncompressed page size, in bytes. More...

chunked_parquet_writer_options_builder &	max_page_size_rows (size_type val)
	Sets the maximum page size, in rows. Counts only top-level rows, ignoring any nesting. Cannot be larger than the row group size in rows, and will be adjusted to match if it is. More...

chunked_parquet_writer_options_builder &	column_index_truncate_length (int32_t val)
	Sets the desired maximum size in bytes for min and max values in the column index. More...

chunked_parquet_writer_options_builder &	dictionary_policy (enum dictionary_policy val)
	Sets the policy for dictionary use. More...

chunked_parquet_writer_options_builder &	max_dictionary_size (size_t val)
	Sets the maximum dictionary size, in bytes. More...

chunked_parquet_writer_options_builder &	max_page_fragment_size (size_type val)
	Sets the maximum page fragment size, in rows. More...

chunked_parquet_writer_options_builder &	compression_statistics (std::shared_ptr< writer_compression_statistics > const &comp_stats)
	Sets the pointer to the output compression statistics. More...

	operator chunked_parquet_writer_options && ()
	move chunked_parquet_writer_options member once it's built.

chunked_parquet_writer_options &&	build ()
	move chunked_parquet_writer_options member once it's is built. More...

Detailed Description

Builds options for chunked_parquet_writer_options.

Definition at line 1517 of file parquet.hpp.

Constructor & Destructor Documentation

◆ chunked_parquet_writer_options_builder() [1/2]

cudf::io::chunked_parquet_writer_options_builder::chunked_parquet_writer_options_builder ( )

default

Default constructor.

This has been added since Cython requires a default constructor to create objects on stack.

◆ chunked_parquet_writer_options_builder() [2/2]

cudf::io::chunked_parquet_writer_options_builder::chunked_parquet_writer_options_builder ( sink_info const & sink )

inline

Constructor from sink.

Parameters

sink	The sink used for writer output

Definition at line 1533 of file parquet.hpp.

Member Function Documentation

◆ build()

chunked_parquet_writer_options&& cudf::io::chunked_parquet_writer_options_builder::build ( )

inline

move chunked_parquet_writer_options member once it's is built.

This has been added since Cython does not support overloading of conversion operators.

Returns: Built chunked_parquet_writer_options object's r-value reference

Definition at line 1755 of file parquet.hpp.

◆ column_index_truncate_length()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::column_index_truncate_length ( int32_t val )

inline

Sets the desired maximum size in bytes for min and max values in the column index.

Values exceeding this limit will be truncated, but modified such that they will still be valid lower and upper bounds. This only applies to variable length types, such as string. Maximum values will not be truncated if there is no suitable truncation that results in a valid upper bound.

Default value is 64.

Parameters

val	length min/max will be truncated to, with 0 indicating no truncation

Returns: this for chaining

Definition at line 1680 of file parquet.hpp.

◆ compression()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::compression ( compression_type compression )

inline

Sets compression type to chunked_parquet_writer_options.

Parameters

compression The compression type to use

Returns: this for chaining

Definition at line 1574 of file parquet.hpp.

◆ compression_statistics()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::compression_statistics ( std::shared_ptr< writer_compression_statistics > const & comp_stats )

inline

Sets the pointer to the output compression statistics.

Parameters

comp_stats Pointer to compression statistics to be filled once writer is done

Returns: this for chaining

Definition at line 1736 of file parquet.hpp.

◆ dictionary_policy()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::dictionary_policy ( enum dictionary_policy val )

Sets the policy for dictionary use.

Certain compression algorithms (e.g Zstandard) have limits on how large of a buffer can be compressed. In some circumstances, the dictionary can grow beyond this limit, which will prevent the column from being compressed. This setting controls how the writer should act in these circumstances. A setting of dictionary_policy::ADAPTIVE will disable dictionary encoding for columns where the dictionary exceeds the limit. A setting of dictionary_policy::NEVER will disable the use of dictionary encoding globally. A setting of dictionary_policy::ALWAYS will allow the use of dictionary encoding even if it will result in the disabling of compression for columns that would otherwise be compressed.

The default value is dictionary_policy::ALWAYS.

Parameters

val	policy for dictionary use

Returns: this for chaining

◆ int96_timestamps()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::int96_timestamps ( bool enabled )

inline

Set to true if timestamps should be written as int96 types instead of int64 types. Even though int96 is deprecated and is not an internal type for cudf, it needs to be written for backwards compatibility reasons.

Parameters

enabled Boolean value to enable/disable int96 timestamps

Returns: this for chaining

Definition at line 1589 of file parquet.hpp.

◆ key_value_metadata()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::key_value_metadata ( std::vector< std::map< std::string, std::string >> metadata )

Sets Key-Value footer metadata in parquet_writer_options.

Parameters

metadata Key-Value footer metadata

Returns: this for chaining

◆ max_dictionary_size()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::max_dictionary_size ( size_t val )

Sets the maximum dictionary size, in bytes.

Disables dictionary encoding for any column chunk where the dictionary will exceed this limit. Only used when the dictionary_policy is set to 'ADAPTIVE'.

Default value is 1048576 (1MiB).

Parameters

val	maximum dictionary size

Returns: this for chaining

◆ max_page_fragment_size()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::max_page_fragment_size ( size_type val )

Sets the maximum page fragment size, in rows.

Files with nested schemas or very long strings may need a page fragment size smaller than the default value of 5000 to ensure a single fragment will not exceed the desired maximum page size in bytes.

Parameters

val	maximum page fragment size

Returns: this for chaining

◆ max_page_size_bytes()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::max_page_size_bytes ( size_t val )

inline

Sets the maximum uncompressed page size, in bytes.

Serves as a hint to the writer, and can be exceeded under certain circumstances. Cannot be larger than the row group size in bytes, and will be adjusted to match if it is.

Parameters

val	maximum page size

Returns: this for chaining

Definition at line 1648 of file parquet.hpp.

◆ max_page_size_rows()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::max_page_size_rows ( size_type val )

inline

Sets the maximum page size, in rows. Counts only top-level rows, ignoring any nesting. Cannot be larger than the row group size in rows, and will be adjusted to match if it is.

Parameters

val	maximum rows per page

Returns: this for chaining

Definition at line 1661 of file parquet.hpp.

◆ metadata()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::metadata ( table_input_metadata metadata )

inline

Sets metadata to chunked_parquet_writer_options.

Parameters

metadata Associated metadata

Returns: this for chaining

Definition at line 1541 of file parquet.hpp.

◆ row_group_size_bytes()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::row_group_size_bytes ( size_t val )

inline

Sets the maximum row group size, in bytes.

Parameters

val	maximum row group size

Returns: this for chaining

Definition at line 1621 of file parquet.hpp.

◆ row_group_size_rows()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::row_group_size_rows ( size_type val )

inline

Sets the maximum number of rows in output row groups.

Parameters

val	maximum number or rows

Returns: this for chaining

Definition at line 1633 of file parquet.hpp.

◆ stats_level()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::stats_level ( statistics_freq sf )

inline

Sets the level of statistics in chunked_parquet_writer_options.

Parameters

sf	Level of statistics requested in the output file

Returns: this for chaining

Definition at line 1562 of file parquet.hpp.

◆ utc_timestamps()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::utc_timestamps ( bool enabled )

inline

Set to true if timestamps are to be written as UTC.

Parameters

enabled Boolean value to enable/disable writing of timestamps as UTC.

Returns: this for chaining

Definition at line 1601 of file parquet.hpp.

◆ write_v2_headers()

chunked_parquet_writer_options_builder& cudf::io::chunked_parquet_writer_options_builder::write_v2_headers ( bool enabled )

Set to true if V2 page headers are to be written.

Parameters

enabled Boolean value to enable/disable writing of V2 page headers.

Returns: this for chaining

The documentation for this class was generated from the following file:

parquet.hpp

Public Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ chunked_parquet_writer_options_builder() [1/2]

◆ chunked_parquet_writer_options_builder() [2/2]

Member Function Documentation

◆ build()

◆ column_index_truncate_length()

◆ compression()

◆ compression_statistics()

◆ dictionary_policy()

◆ int96_timestamps()

◆ key_value_metadata()

◆ max_dictionary_size()

◆ max_page_fragment_size()

◆ max_page_size_bytes()

◆ max_page_size_rows()

◆ metadata()

◆ row_group_size_bytes()

◆ row_group_size_rows()

◆ stats_level()

◆ utc_timestamps()

◆ write_v2_headers()