Namespaces
namespace	utils

Classes
class	any_output_stream
	Type-erased wrapper for any stream-like object. More...

class	arrow_array_private_data

class	chunk_serializer
	A serializer that writes record batches to chunked memory streams. More...

class	chunked_memory_output_stream
	An output stream that writes data into separate memory chunks. More...

class	CompressionCache

class	deserializer

class	dictionary_cache
	Caches dictionaries during deserialization. More...

struct	dictionary_info
	Information about a dictionary used for encoding. More...

struct	dictionary_metadata

class	dictionary_tracker
	Tracks dictionaries during serialization. More...

class	memory_output_stream
	An output stream that writes data to a contiguous memory buffer. More...

class	non_owning_arrow_schema_private_data

struct	record_batch_block
	Represents a block entry in the Arrow IPC file footer. More...

struct	record_batch_stream
	Result of stream deserialization containing optional schema and record batches. More...

struct	serialized_record_batch_info
	Information about a serialized record batch block. More...

class	serializer
	A class for serializing Apache Arrow record batches to an output stream. More...

class	stream_file_serializer
	A class for serializing Apache Arrow record batches to the IPC file format. More...

Concepts
concept	writable_stream
	Concept for stream-like types that support write operations.

concept	ArrowPrivateData

Enumerations
enum class	CompressionType : std::uint8_t { LZ4_FRAME , ZSTD }

Functions
SPARROW_IPC_API void	release_arrow_array_children_and_dictionary (ArrowArray *array)

template<ArrowPrivateData T>
void	arrow_array_release (ArrowArray *array)

template<ArrowPrivateData T, typename Arg>
void	fill_arrow_array (ArrowArray &array, int64_t length, int64_t null_count, int64_t offset, size_t children_count, ArrowArray *children, ArrowArray dictionary, Arg &&private_data_arg)

template<ArrowPrivateData T, typename Arg>
ArrowArray	make_arrow_array (int64_t length, int64_t null_count, int64_t offset, size_t children_count, ArrowArray *children, ArrowArray dictionary, Arg &&private_data_arg)

template<class T> requires std::same_as<T, ArrowArray> \|\| std::same_as<T, ArrowSchema>
void	release_common_non_owning_arrow (T &t)
	Release the children and dictionnary of an `ArrowArray` or `ArrowSchema`.

SPARROW_IPC_API void	release_non_owning_arrow_schema (ArrowSchema *schema)

template<sparrow::input_metadata_container M = std::vector<sparrow::metadata_pair>>
void	fill_non_owning_arrow_schema (ArrowSchema &schema, std::string_view format, std::string_view name, std::optional< M > metadata, std::optional< std::unordered_set< sparrow::ArrowFlag > > flags, size_t children_count, ArrowSchema *children, ArrowSchema dictionary)

template<sparrow::input_metadata_container M = std::vector<sparrow::metadata_pair>>
ArrowSchema	make_non_owning_arrow_schema (std::string_view format, std::string_view name, std::optional< M > metadata, std::optional< std::unordered_set< sparrow::ArrowFlag > > flags, size_t children_count, ArrowSchema *children, ArrowSchema dictionary)

SPARROW_IPC_API std::span< const std::uint8_t >	compress (const CompressionType compression_type, const std::span< const std::uint8_t > &data, CompressionCache &cache)

SPARROW_IPC_API size_t	get_compressed_size (const CompressionType compression_type, const std::span< const std::uint8_t > &data, CompressionCache &cache)

SPARROW_IPC_API std::variant< sparrow::buffer< std::uint8_t >, std::span< const std::uint8_t > >	decompress (const CompressionType compression_type, std::span< const std::uint8_t > data)

sparrow::buffer< uint8_t >	make_zeroed_buffer (std::size_t size)

template<class T>
sparrow::buffer< uint8_t >	make_typed_zeroed_buffer (std::size_t count)

template<class T>
sparrow::buffer< uint8_t >	make_zero_offset_buffer ()

template<typename CHILD>
CHILD **	make_owned_children (std::vector< CHILD > &&children)

SPARROW_IPC_API record_batch_stream	deserialize_stream_to_record_batches (std::span< const uint8_t > data)
	Deserializes an Arrow IPC stream into a record_batch_stream.

SPARROW_IPC_API std::vector< sparrow::record_batch >	deserialize_stream (std::span< const uint8_t > data)
	Deserializes an Arrow IPC stream from binary data into a vector of record batches.

SPARROW_IPC_API record_batch_stream	deserialize_file (std::span< const uint8_t > data)
	Deserializes Arrow IPC file format into a record_batch_stream.

template<class Func>
void	for_each_pending_dictionary (const sparrow::record_batch &record_batch, dictionary_tracker &tracker, Func visitor)

SPARROW_IPC_API int64_t	compute_fallback_dictionary_id (std::string_view field_name, size_t field_index)

SPARROW_IPC_API dictionary_metadata	parse_dictionary_metadata (const ArrowSchema &schema)

template<std::ranges::input_range R>
bool	is_continuation (const R &buf)

template<std::ranges::input_range R>
bool	is_end_of_stream (const R &buf)

template<std::ranges::input_range R>
bool	is_arrow_file_magic (const R &buf)

template<std::ranges::input_range R> requires std::same_as<std::ranges::range_value_t<R>, sparrow::record_batch>
void	serialize_record_batches_to_ipc_stream (const R &record_batches, any_output_stream &stream, std::optional< CompressionType > compression, std::optional< std::reference_wrapper< CompressionCache > > cache)
	Serializes a collection of record batches into a binary format.

SPARROW_IPC_API serialized_record_batch_info	serialize_record_batch (const sparrow::record_batch &record_batch, any_output_stream &stream, std::optional< CompressionType > compression, std::optional< std::reference_wrapper< CompressionCache > > cache)
	Serializes a record batch into a binary format following the Arrow IPC specification.

SPARROW_IPC_API serialized_record_batch_info	serialize_dictionary_batch (int64_t dictionary_id, const sparrow::record_batch &record_batch, bool is_delta, any_output_stream &stream, std::optional< CompressionType > compression, std::optional< std::reference_wrapper< CompressionCache > > cache)
	Serializes a dictionary batch into a binary format following the Arrow IPC specification.

SPARROW_IPC_API void	serialize_schema_message (const sparrow::record_batch &record_batch, any_output_stream &stream)
	Serializes a schema message for a record batch into a byte buffer.

SPARROW_IPC_API std::size_t	calculate_schema_message_size (const sparrow::record_batch &record_batch)
	Calculates the total serialized size of a schema message.

SPARROW_IPC_API std::size_t	calculate_record_batch_message_size (const sparrow::record_batch &record_batch, std::optional< CompressionType > compression=std::nullopt, std::optional< std::reference_wrapper< CompressionCache > > cache=std::nullopt)
	Calculates the total serialized size of a record batch message.

SPARROW_IPC_API std::size_t	calculate_dictionary_batch_message_size (int64_t dictionary_id, const sparrow::record_batch &record_batch, bool is_delta, std::optional< CompressionType > compression=std::nullopt, std::optional< std::reference_wrapper< CompressionCache > > cache=std::nullopt)
	Calculates the total serialized size of a dictionary batch message.

template<std::ranges::input_range R> requires std::same_as<std::ranges::range_value_t<R>, sparrow::record_batch>
std::size_t	calculate_total_serialized_size (const R &record_batches, std::optional< CompressionType > compression=std::nullopt, std::optional< std::reference_wrapper< CompressionCache > > cache=std::nullopt)
	Calculates the total serialized size for a collection of record batches.

SPARROW_IPC_API void	fill_body (const sparrow::arrow_proxy &arrow_proxy, any_output_stream &stream, std::optional< CompressionType > compression=std::nullopt, std::optional< std::reference_wrapper< CompressionCache > > cache=std::nullopt)
	Fills the body vector with serialized data from an arrow proxy and its children.

SPARROW_IPC_API void	generate_body (const sparrow::record_batch &record_batch, any_output_stream &stream, std::optional< CompressionType > compression=std::nullopt, std::optional< std::reference_wrapper< CompressionCache > > cache=std::nullopt)
	Generates a serialized body from a record batch.

SPARROW_IPC_API std::vector< sparrow::data_type >	get_column_dtypes (const sparrow::record_batch &rb)

serializer &	end_stream (serializer &serializer)

template<std::ranges::input_range R> requires std::same_as<std::ranges::range_value_t<R>, sparrow::record_batch>
std::size_t	calculate_serializer_reserve_size (const R &record_batches, std::size_t current_stream_size, bool schema_received, std::optional< CompressionType > compression, const dictionary_tracker &dict_tracker, std::optional< std::reference_wrapper< CompressionCache > > cache=std::nullopt)

SPARROW_IPC_API size_t	write_footer (const sparrow::record_batch &record_batch, const std::vector< record_batch_block > &dictionary_blocks, const std::vector< record_batch_block > &record_batch_blocks, any_output_stream &stream)
	Writes the Arrow IPC file footer.

stream_file_serializer &	end_file (stream_file_serializer &serializer)

Variables
constexpr int	SPARROW_IPC_VERSION_MAJOR = 2

constexpr int	SPARROW_IPC_VERSION_MINOR = 0

constexpr int	SPARROW_IPC_VERSION_PATCH = 0

constexpr int	SPARROW_IPC_BINARY_CURRENT = 5

constexpr int	SPARROW_IPC_BINARY_REVISION = 0

constexpr int	SPARROW_IPC_BINARY_AGE = 0

constexpr std::array< std::uint8_t, 4 >	continuation = {0xFF, 0xFF, 0xFF, 0xFF}
	Continuation value defined in the Arrow IPC specification: https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format.

constexpr std::array< std::uint8_t, 8 >	end_of_stream = {0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00}
	End-of-stream marker defined in the Arrow IPC specification: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format.

constexpr std::array< std::uint8_t, 6 >	arrow_file_magic = {'A', 'R', 'R', 'O', 'W', '1'}
	Magic bytes for Arrow file format defined in the Arrow IPC specification: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format The magic string is "ARROW1" (6 bytes) followed by 2 padding bytes to reach 8-byte alignment.

constexpr std::size_t	arrow_file_magic_size = arrow_file_magic.size()

constexpr std::array< std::uint8_t, 8 >	arrow_file_header_magic = {'A', 'R', 'R', 'O', 'W', '1', 0x00, 0x00}
	Magic bytes with padding for file header (8 bytes total for alignment)

Enumeration Type Documentation

◆ CompressionType

enum class sparrow_ipc::CompressionType : std::uint8_t

strong

Enumerator
LZ4_FRAME
ZSTD

Definition at line 16 of file compression.hpp.

Function Documentation

◆ arrow_array_release()

template<ArrowPrivateData T>

void sparrow_ipc::arrow_array_release ( ArrowArray * array )

Definition at line 16 of file arrow_array.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ calculate_dictionary_batch_message_size()

SPARROW_IPC_API std::size_t sparrow_ipc::calculate_dictionary_batch_message_size	(	int64_t	dictionary_id,
		const sparrow::record_batch &	record_batch,
		bool	is_delta,
		std::optional< CompressionType >	compression = std::nullopt,
		std::optional< std::reference_wrapper< CompressionCache > >	cache = std::nullopt )

nodiscard

Calculates the total serialized size of a dictionary batch message.

This function computes the complete size that would be produced by serialize_dictionary_batch(), including:

Continuation bytes (4 bytes)
Message length prefix (4 bytes)
FlatBuffer dictionary batch metadata
Padding to 8-byte alignment after metadata
Body data with 8-byte alignment between buffers

Parameters

dictionary_id	The unique identifier for the dictionary.
record_batch	A single-column record batch containing dictionary values.
is_delta	Whether this dictionary batch is a delta update.
compression	Optional: The compression type to use when serializing.
cache	Optional: A cache to store and retrieve compressed buffer sizes, avoiding recompression. If compression is given, cache should be set as well.

Returns: The total size in bytes that the serialized dictionary batch would occupy.

◆ calculate_record_batch_message_size()

SPARROW_IPC_API std::size_t sparrow_ipc::calculate_record_batch_message_size	(	const sparrow::record_batch &	record_batch,
		std::optional< CompressionType >	compression = std::nullopt,
		std::optional< std::reference_wrapper< CompressionCache > >	cache = std::nullopt )

nodiscard

Calculates the total serialized size of a record batch message.

This function computes the complete size that would be produced by serialize_record_batch(), including:

Continuation bytes (4 bytes)
Message length prefix (4 bytes)
FlatBuffer record batch metadata
Padding to 8-byte alignment after metadata
Body data with 8-byte alignment between buffers

Parameters

record_batch	The record batch to be measured.
compression	Optional: The compression type to use when serializing.
cache	Optional: A cache to store and retrieve compressed buffer sizes, avoiding recompression. If compression is given, cache should be set as well.

Returns: The total size in bytes that the serialized record batch would occupy.

Here is the caller graph for this function:

◆ calculate_schema_message_size()

SPARROW_IPC_API std::size_t sparrow_ipc::calculate_schema_message_size ( const sparrow::record_batch & record_batch )

nodiscard

Calculates the total serialized size of a schema message.

This function computes the complete size that would be produced by serialize_schema_message(), including:

Continuation bytes (4 bytes)
Message length prefix (4 bytes)
FlatBuffer schema message data
Padding to 8-byte alignment

Parameters

record_batch The record batch containing the schema to be measured

Returns: The total size in bytes that the serialized schema message would occupy

Here is the caller graph for this function:

◆ calculate_serializer_reserve_size()

template<std::ranges::input_range R>
requires std::same_as<std::ranges::range_value_t<R>, sparrow::record_batch>

std::size_t sparrow_ipc::calculate_serializer_reserve_size	(	const R &	record_batches,
		std::size_t	current_stream_size,
		bool	schema_received,
		std::optional< CompressionType >	compression,
		const dictionary_tracker &	dict_tracker,
		std::optional< std::reference_wrapper< CompressionCache > >	cache = std::nullopt )

nodiscard

Definition at line 17 of file serializer_reserve.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ calculate_total_serialized_size()

template<std::ranges::input_range R>
requires std::same_as<std::ranges::range_value_t<R>, sparrow::record_batch>

std::size_t sparrow_ipc::calculate_total_serialized_size	(	const R &	record_batches,
		std::optional< CompressionType >	compression = std::nullopt,
		std::optional< std::reference_wrapper< CompressionCache > >	cache = std::nullopt )

nodiscard

Calculates the total serialized size for a collection of record batches.

This function computes the complete size that would be produced by serializing a schema message followed by all record batch messages in the collection.

Template Parameters

R	Range type containing sparrow::record_batch objects.

Parameters

record_batches	Collection of record batches to be measured.
compression	Optional: The compression type to use when serializing.
cache	Optional: A cache to store and retrieve compressed buffer sizes, avoiding recompression. If compression is given, cache should be set as well.

Returns: The total size in bytes for the complete serialized output.

Exceptions

std::invalid_argument if record batches have inconsistent schemas.

Definition at line 110 of file serialize_utils.hpp.

Here is the call graph for this function:

◆ compress()

SPARROW_IPC_API std::span< const std::uint8_t > sparrow_ipc::compress	(	const CompressionType	compression_type,
		const std::span< const std::uint8_t > &	data,
		CompressionCache &	cache )

nodiscard

Here is the call graph for this function:

Here is the caller graph for this function:

◆ compute_fallback_dictionary_id()

SPARROW_IPC_API int64_t sparrow_ipc::compute_fallback_dictionary_id	(	std::string_view	field_name,
		size_t	field_index )

nodiscard

◆ decompress()

SPARROW_IPC_API std::variant< sparrow::buffer< std::uint8_t >, std::span< const std::uint8_t > > sparrow_ipc::decompress	(	const CompressionType	compression_type,
		std::span< const std::uint8_t >	data )

nodiscard

Here is the call graph for this function:

Here is the caller graph for this function:

◆ deserialize_file()

SPARROW_IPC_API record_batch_stream sparrow_ipc::deserialize_file ( std::span< const uint8_t > data )

nodiscard

Deserializes Arrow IPC file format into a record_batch_stream.

Reads an Arrow IPC file format which consists of:

Magic bytes "ARROW1" with padding (8 bytes)
Stream format data (schema + record batches)
Footer containing metadata
Footer size (int32)
Trailing magic bytes "ARROW1" (6 bytes)

Parameters

data	A span of bytes containing the serialized Arrow IPC file data

Returns: record_batch_stream Containing schema and batches

Exceptions

std::runtime_error

If:

The file magic bytes are incorrect
The footer is missing or invalid
Record batch deserialization fails

Note: The function validates the file structure including magic bytes at both start and end

◆ deserialize_stream()

SPARROW_IPC_API std::vector< sparrow::record_batch > sparrow_ipc::deserialize_stream ( std::span< const uint8_t > data )

nodiscard

Deserializes an Arrow IPC stream from binary data into a vector of record batches.

... This function processes an Arrow IPC stream format, extracting schema information and record batch data. It handles encapsulated messages sequentially, first expecting a Schema message followed by one or more RecordBatch messages.

Parameters

data	A span of bytes containing the serialized Arrow IPC stream data

Returns: std::vector<sparrow::record_batch> A vector containing all deserialized record batches

Exceptions

std::runtime_error

If:

A RecordBatch message is encountered before a Schema message
A RecordBatch message header is missing or invalid
Unsupported message types are encountered (Tensor, DictionaryBatch, SparseTensor)
An unknown message header type is encountered

Note: The function processes messages until an end-of-stream marker is detected

Here is the caller graph for this function:

◆ deserialize_stream_to_record_batches()

SPARROW_IPC_API record_batch_stream sparrow_ipc::deserialize_stream_to_record_batches ( std::span< const uint8_t > data )

nodiscard

Deserializes an Arrow IPC stream into a record_batch_stream.

This function returns both the schema (as an optional record batch) and the record batches found in the stream.

Parameters

data	A span of bytes containing the serialized Arrow IPC stream data

Returns: record_batch_stream Containing schema and batches

◆ end_file()

stream_file_serializer & sparrow_ipc::end_file ( stream_file_serializer & serializer )

inline

Examples: /home/runner/work/sparrow-ipc/sparrow-ipc/include/sparrow_ipc/stream_file_serializer.hpp.

Definition at line 360 of file stream_file_serializer.hpp.

Here is the call graph for this function:

◆ end_stream()

serializer & sparrow_ipc::end_stream ( serializer & serializer )

inline

Examples: /home/runner/work/sparrow-ipc/sparrow-ipc/include/sparrow_ipc/serializer.hpp.

Definition at line 265 of file serializer.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fill_arrow_array()

template<ArrowPrivateData T, typename Arg>

void sparrow_ipc::fill_arrow_array	(	ArrowArray &	array,
		int64_t	length,
		int64_t	null_count,
		int64_t	offset,
		size_t	children_count,
		ArrowArray **	children,
		ArrowArray *	dictionary,
		Arg &&	private_data_arg )

Definition at line 32 of file arrow_array.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fill_body()

SPARROW_IPC_API void sparrow_ipc::fill_body	(	const sparrow::arrow_proxy &	arrow_proxy,
		any_output_stream &	stream,
		std::optional< CompressionType >	compression = std::nullopt,
		std::optional< std::reference_wrapper< CompressionCache > >	cache = std::nullopt )

Fills the body vector with serialized data from an arrow proxy and its children.

This function recursively processes an arrow proxy by:

Iterating through all buffers in the proxy and appending their data to the body vector
Adding padding bytes (zeros) after each buffer to align data to 8-byte boundaries
Recursively processing all child proxies in the same manner

The function ensures proper memory alignment by padding each buffer's data to the next 8-byte boundary, which is typically required for efficient memory access and Arrow format compliance.

Parameters

arrow_proxy	The arrow proxy containing buffers and potential child proxies to serialize.
stream	The output stream where the serialized body data will be written.
compression	Optional: The compression type to use when serializing.
cache	Optional: A cache for compressed buffers to avoid recompression if compression is enabled. If compression is given, cache should be set as well.

Exceptions

std::invalid_argument if compression is given but not cache.

◆ fill_non_owning_arrow_schema()

template<sparrow::input_metadata_container M = std::vector<sparrow::metadata_pair>>

void sparrow_ipc::fill_non_owning_arrow_schema	(	ArrowSchema &	schema,
		std::string_view	format,
		std::string_view	name,
		std::optional< M >	metadata,
		std::optional< std::unordered_set< sparrow::ArrowFlag > >	flags,
		size_t	children_count,
		ArrowSchema **	children,
		ArrowSchema *	dictionary )

Definition at line 18 of file arrow_schema.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ for_each_pending_dictionary()

template<class Func>

void sparrow_ipc::for_each_pending_dictionary	(	const sparrow::record_batch &	record_batch,
		dictionary_tracker &	tracker,
		Func	visitor )

Definition at line 11 of file dictionary_iteration.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ generate_body()

SPARROW_IPC_API void sparrow_ipc::generate_body	(	const sparrow::record_batch &	record_batch,
		any_output_stream &	stream,
		std::optional< CompressionType >	compression = std::nullopt,
		std::optional< std::reference_wrapper< CompressionCache > >	cache = std::nullopt )

Generates a serialized body from a record batch.

This function iterates through all columns in the provided record batch, extracts their Arrow proxy representations, and serializes them into a single byte vector that forms the body of the serialized data.

Parameters

record_batch	The record batch containing columns to be serialized.
stream	The output stream where the serialized body will be written.
compression	Optional: The compression type to use when serializing.
cache	Optional: A cache for compressed buffers to avoid recompression if compression is enabled. If compression is given, cache should be set as well.

◆ get_column_dtypes()

SPARROW_IPC_API std::vector< sparrow::data_type > sparrow_ipc::get_column_dtypes ( const sparrow::record_batch & rb )

Examples: /home/runner/work/sparrow-ipc/sparrow-ipc/include/sparrow_ipc/chunk_memory_serializer.hpp.

Here is the caller graph for this function:

◆ get_compressed_size()

SPARROW_IPC_API size_t sparrow_ipc::get_compressed_size	(	const CompressionType	compression_type,
		const std::span< const std::uint8_t > &	data,
		CompressionCache &	cache )

nodiscard

Here is the call graph for this function:

Here is the caller graph for this function:

◆ is_arrow_file_magic()

template<std::ranges::input_range R>

bool sparrow_ipc::is_arrow_file_magic ( const R & buf )

nodiscard

Definition at line 49 of file magic_values.hpp.

◆ is_continuation()

template<std::ranges::input_range R>

bool sparrow_ipc::is_continuation ( const R & buf )

nodiscard

Definition at line 37 of file magic_values.hpp.

◆ is_end_of_stream()

template<std::ranges::input_range R>

bool sparrow_ipc::is_end_of_stream ( const R & buf )

nodiscard

Definition at line 43 of file magic_values.hpp.

◆ make_arrow_array()

template<ArrowPrivateData T, typename Arg>

ArrowArray sparrow_ipc::make_arrow_array	(	int64_t	length,
		int64_t	null_count,
		int64_t	offset,
		size_t	children_count,
		ArrowArray **	children,
		ArrowArray *	dictionary,
		Arg &&	private_data_arg )

nodiscard

Definition at line 63 of file arrow_array.hpp.

Here is the call graph for this function:

◆ make_non_owning_arrow_schema()

template<sparrow::input_metadata_container M = std::vector<sparrow::metadata_pair>>

ArrowSchema sparrow_ipc::make_non_owning_arrow_schema	(	std::string_view	format,
		std::string_view	name,
		std::optional< M >	metadata,
		std::optional< std::unordered_set< sparrow::ArrowFlag > >	flags,
		size_t	children_count,
		ArrowSchema **	children,
		ArrowSchema *	dictionary )

nodiscard

Definition at line 57 of file arrow_schema.hpp.

Here is the call graph for this function:

◆ make_owned_children()

template<typename CHILD>

CHILD ** sparrow_ipc::make_owned_children ( std::vector< CHILD > && children )

nodiscard

Definition at line 32 of file deserialize.hpp.

◆ make_typed_zeroed_buffer()

template<class T>

sparrow::buffer< uint8_t > sparrow_ipc::make_typed_zeroed_buffer ( std::size_t count )

nodiscard

Definition at line 20 of file deserialize.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ make_zero_offset_buffer()

template<class T>

sparrow::buffer< uint8_t > sparrow_ipc::make_zero_offset_buffer ( )

nodiscard

Definition at line 26 of file deserialize.hpp.

Here is the call graph for this function:

◆ make_zeroed_buffer()

sparrow::buffer< uint8_t > sparrow_ipc::make_zeroed_buffer ( std::size_t size )

inlinenodiscard

Definition at line 14 of file deserialize.hpp.

Here is the caller graph for this function:

◆ parse_dictionary_metadata()

SPARROW_IPC_API dictionary_metadata sparrow_ipc::parse_dictionary_metadata ( const ArrowSchema & schema )

nodiscard

◆ release_arrow_array_children_and_dictionary()

SPARROW_IPC_API void sparrow_ipc::release_arrow_array_children_and_dictionary ( ArrowArray * array )

Here is the caller graph for this function:

◆ release_common_non_owning_arrow()

template<class T>
requires std::same_as<T, ArrowArray> || std::same_as<T, ArrowSchema>

void sparrow_ipc::release_common_non_owning_arrow ( T & t )

Release the children and dictionnary of an ArrowArray or ArrowSchema.

Template Parameters

T	`ArrowArray` or `ArrowSchema`

Parameters

t	The `ArrowArray` or `ArrowSchema` to release.

Definition at line 20 of file arrow_array_schema_common_release.hpp.

◆ release_non_owning_arrow_schema()

SPARROW_IPC_API void sparrow_ipc::release_non_owning_arrow_schema ( ArrowSchema * schema )

Here is the caller graph for this function:

◆ serialize_dictionary_batch()

SPARROW_IPC_API serialized_record_batch_info sparrow_ipc::serialize_dictionary_batch	(	int64_t	dictionary_id,
		const sparrow::record_batch &	record_batch,
		bool	is_delta,
		any_output_stream &	stream,
		std::optional< CompressionType >	compression,
		std::optional< std::reference_wrapper< CompressionCache > >	cache )

Serializes a dictionary batch into a binary format following the Arrow IPC specification.

This function serializes a dictionary batch containing the actual values referenced by dictionary-encoded fields. The dictionary data is provided as a single-column record batch. The serialized output follows the Arrow IPC encapsulated message format.

Parameters

dictionary_id	The unique identifier for this dictionary, used to match with dictionary-encoded fields in record batches
record_batch	A single-column record batch containing the dictionary values
is_delta	If true, this dictionary should be appended to an existing dictionary with the same ID. If false, it replaces any existing dictionary.
stream	The output stream where the serialized dictionary batch will be written
compression	Optional: The compression type to use when serializing
cache	Optional: A cache to store and retrieve compressed buffers, avoiding recompression. If compression is given, cache should be set as well.

Returns: Information about the serialized dictionary batch (metadata and body lengths)

Exceptions

std::invalid_argument if record_batch doesn't have exactly one column

Note: Dictionary batches must be emitted before any RecordBatch that references them

Here is the caller graph for this function:

◆ serialize_record_batch()

SPARROW_IPC_API serialized_record_batch_info sparrow_ipc::serialize_record_batch	(	const sparrow::record_batch &	record_batch,
		any_output_stream &	stream,
		std::optional< CompressionType >	compression,
		std::optional< std::reference_wrapper< CompressionCache > >	cache )

Serializes a record batch into a binary format following the Arrow IPC specification.

This function converts a sparrow record batch into a serialized byte vector that includes:

A continuation marker
The record batch message length (4 bytes)
The flatbuffer-encoded record batch metadata
Padding to align to 8-byte boundaries
The record batch body containing the actual data buffers

Parameters

record_batch	The sparrow record batch to serialize
stream	The output stream where the serialized record batch will be written
compression	Optional: The compression type to use when serializing.
cache	Optional: A cache to store and retrieve compressed buffers, avoiding recompression.

Note: If compression is given, cache should be set as well.; The output follows Arrow IPC message format with proper alignment and includes both metadata and data portions of the record batch

Examples: /home/runner/work/sparrow-ipc/sparrow-ipc/include/sparrow_ipc/chunk_memory_serializer.hpp.

Here is the caller graph for this function:

◆ serialize_record_batches_to_ipc_stream()

template<std::ranges::input_range R>
requires std::same_as<std::ranges::range_value_t<R>, sparrow::record_batch>

void sparrow_ipc::serialize_record_batches_to_ipc_stream	(	const R &	record_batches,
		any_output_stream &	stream,
		std::optional< CompressionType >	compression,
		std::optional< std::reference_wrapper< CompressionCache > >	cache )

Serializes a collection of record batches into a binary format.

This function takes a collection of record batches and serializes them into a single binary representation following the Arrow IPC format. The serialization includes:

Schema message (derived from the first record batch)
All record batch data
End-of-stream marker

Template Parameters

R	Container type that holds record batches (must support empty(), operator[], begin(), end())

Parameters

record_batches	Collection of record batches to serialize. All batches must have identical schemas.
stream	The output stream where the serialized data will be written.
compression	Optional: The compression type to use when serializing.
cache	Optional: A cache to store and retrieve compressed buffers, avoiding recompression. If compression is given, cache should be set as well.

Exceptions

std::invalid_argument If record batches have inconsistent schemas or if the collection contains batches that cannot be serialized together.

Precondition: All record batches in the collection must have the same schema; The container R must not be empty when consistency checking is required

Definition at line 50 of file serialize.hpp.

Here is the call graph for this function:

◆ serialize_schema_message()

SPARROW_IPC_API void sparrow_ipc::serialize_schema_message	(	const sparrow::record_batch &	record_batch,
		any_output_stream &	stream )

Serializes a schema message for a record batch into a byte buffer.

Serializes a record batch schema into a binary message format.

This function creates a serialized schema message following the Arrow IPC format. The resulting buffer contains:

Continuation bytes at the beginning
A 4-byte length prefix indicating the size of the schema message
The actual FlatBuffer schema message bytes
Padding bytes to align the total size to 8-byte boundaries

Parameters

record_batch	The record batch containing the schema to serialize
stream	The output stream where the serialized schema message will be written

This function creates a serialized schema message by combining continuation bytes, a length prefix, the flatbuffer schema data, and padding to ensure 8-byte alignment. The resulting format follows the Arrow IPC specification for schema messages.

Parameters

record_batch	The record batch containing the schema to be serialized
stream	The output stream where the serialized schema message will be written

Examples: /home/runner/work/sparrow-ipc/sparrow-ipc/include/sparrow_ipc/chunk_memory_serializer.hpp.

Here is the caller graph for this function:

◆ write_footer()

SPARROW_IPC_API size_t sparrow_ipc::write_footer	(	const sparrow::record_batch &	record_batch,
		const std::vector< record_batch_block > &	dictionary_blocks,
		const std::vector< record_batch_block > &	record_batch_blocks,
		any_output_stream &	stream )

Writes the Arrow IPC file footer.

Parameters

record_batch	A record batch containing the schema for the footer
record_batch_blocks	Vector of block information for each record batch
stream	The output stream to write the footer to

Returns: The size of the footer in bytes

Examples: /home/runner/work/sparrow-ipc/sparrow-ipc/include/sparrow_ipc/stream_file_serializer.hpp.

Variable Documentation

◆ arrow_file_header_magic

std::array<std::uint8_t, 8> sparrow_ipc::arrow_file_header_magic = {'A', 'R', 'R', 'O', 'W', '1', 0x00, 0x00}

inlineconstexpr

Magic bytes with padding for file header (8 bytes total for alignment)

Definition at line 34 of file magic_values.hpp.

◆ arrow_file_magic

std::array<std::uint8_t, 6> sparrow_ipc::arrow_file_magic = {'A', 'R', 'R', 'O', 'W', '1'}

inlineconstexpr

Magic bytes for Arrow file format defined in the Arrow IPC specification: https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format The magic string is "ARROW1" (6 bytes) followed by 2 padding bytes to reach 8-byte alignment.

Definition at line 28 of file magic_values.hpp.

◆ arrow_file_magic_size

std::size_t sparrow_ipc::arrow_file_magic_size = arrow_file_magic.size()

inlineconstexpr

Definition at line 29 of file magic_values.hpp.

◆ continuation

std::array<std::uint8_t, 4> sparrow_ipc::continuation = {0xFF, 0xFF, 0xFF, 0xFF}

inlineconstexpr

Continuation value defined in the Arrow IPC specification: https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format.

Definition at line 15 of file magic_values.hpp.

◆ end_of_stream

std::array<std::uint8_t, 8> sparrow_ipc::end_of_stream = {0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00}

inlineconstexpr

End-of-stream marker defined in the Arrow IPC specification: https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format.

Definition at line 21 of file magic_values.hpp.

◆ SPARROW_IPC_BINARY_AGE

int sparrow_ipc::SPARROW_IPC_BINARY_AGE = 0

constexpr

Definition at line 14 of file sparrow_ipc_version.hpp.

◆ SPARROW_IPC_BINARY_CURRENT

int sparrow_ipc::SPARROW_IPC_BINARY_CURRENT = 5

constexpr

Definition at line 12 of file sparrow_ipc_version.hpp.

◆ SPARROW_IPC_BINARY_REVISION

int sparrow_ipc::SPARROW_IPC_BINARY_REVISION = 0

constexpr

Definition at line 13 of file sparrow_ipc_version.hpp.

◆ SPARROW_IPC_VERSION_MAJOR

int sparrow_ipc::SPARROW_IPC_VERSION_MAJOR = 2

constexpr

Definition at line 5 of file sparrow_ipc_version.hpp.

◆ SPARROW_IPC_VERSION_MINOR

int sparrow_ipc::SPARROW_IPC_VERSION_MINOR = 0

constexpr

Definition at line 6 of file sparrow_ipc_version.hpp.

◆ SPARROW_IPC_VERSION_PATCH

int sparrow_ipc::SPARROW_IPC_VERSION_PATCH = 0

constexpr

Definition at line 7 of file sparrow_ipc_version.hpp.

Namespaces

Classes

Concepts

Enumerations

Functions

Variables

Enumeration Type Documentation

◆ CompressionType

Function Documentation

◆ arrow_array_release()

◆ calculate_dictionary_batch_message_size()

◆ calculate_record_batch_message_size()

◆ calculate_schema_message_size()

◆ calculate_serializer_reserve_size()

◆ calculate_total_serialized_size()

◆ compress()

◆ compute_fallback_dictionary_id()

◆ decompress()

◆ deserialize_file()

◆ deserialize_stream()

◆ deserialize_stream_to_record_batches()

◆ end_file()

◆ end_stream()

◆ fill_arrow_array()

◆ fill_body()

◆ fill_non_owning_arrow_schema()

◆ for_each_pending_dictionary()

◆ generate_body()

◆ get_column_dtypes()

◆ get_compressed_size()

◆ is_arrow_file_magic()

◆ is_continuation()

◆ is_end_of_stream()

◆ make_arrow_array()

◆ make_non_owning_arrow_schema()

◆ make_owned_children()

◆ make_typed_zeroed_buffer()

◆ make_zero_offset_buffer()

◆ make_zeroed_buffer()

◆ parse_dictionary_metadata()

◆ release_arrow_array_children_and_dictionary()

◆ release_common_non_owning_arrow()

◆ release_non_owning_arrow_schema()

◆ serialize_dictionary_batch()

◆ serialize_record_batch()

◆ serialize_record_batches_to_ipc_stream()

◆ serialize_schema_message()

◆ write_footer()

Variable Documentation

◆ arrow_file_header_magic

◆ arrow_file_magic

◆ arrow_file_magic_size

◆ continuation

◆ end_of_stream

◆ SPARROW_IPC_BINARY_AGE

◆ SPARROW_IPC_BINARY_CURRENT

◆ SPARROW_IPC_BINARY_REVISION

◆ SPARROW_IPC_VERSION_MAJOR

◆ SPARROW_IPC_VERSION_MINOR

◆ SPARROW_IPC_VERSION_PATCH