API for appending new files in a table. More...

#include <fast_append.h>

Inheritance diagram for iceberg::FastAppend:

Public Member Functions
FastAppend &	AppendFile (const std::shared_ptr< DataFile > &file)
	Append a DataFile to the table.

FastAppend &	AppendManifest (const ManifestFile &manifest)
	Append a ManifestFile to the table.

std::string	operation () override
	A string that describes the action that produced the new snapshot.

Result< std::vector< ManifestFile > >	Apply (const TableMetadata &metadata_to_update, const std::shared_ptr< Snapshot > &snapshot) override
	Apply the update's changes to the given metadata and snapshot.

std::unordered_map< std::string, std::string >	Summary () override
	Get the summary map for this operation.

void	SetSummaryProperty (const std::string &property, const std::string &value) override
	Set a summary property.

Status	CleanUncommitted (const std::unordered_set< std::string > &committed) override
	Clean up any uncommitted manifests that were created.

bool	CleanupAfterCommit () const override
	Check if cleanup should happen after commit.

Public Member Functions inherited from iceberg::SnapshotUpdate
Kind	kind () const override
	Return the kind of this pending update.

bool	IsRetryable () const override
	Whether this update can be retried after a commit conflict.

auto &	DeleteWith (this auto &self, std::function< Status(const std::string &)> delete_func)
	Set a callback to delete files instead of the table's default.

auto &	StageOnly (this auto &self)
	Stage a snapshot in table metadata, but do not make it current.

auto &	ScanManifestsWith (this auto &self, Executor &executor)
	Configure an executor for manifest planning work.

auto &	ToBranch (this auto &self, const std::string &branch)
	Perform operations on a particular branch.

auto &	Set (this auto &self, const std::string &property, const std::string &value)
	Set a summary property in the snapshot produced by this update.

auto &	WriteManifestsWith (this auto &self, Executor &executor, int32_t parallelism)
	Configure an executor and max writer count for writing new manifests.

Result< ApplyResult >	Apply ()
	Apply the update's changes to create a new snapshot.

Status	Finalize (Result< const TableMetadata * > commit_result) override
	Finalize the snapshot update, cleaning up any uncommitted files.

Public Member Functions inherited from iceberg::PendingUpdate
virtual Status	Commit ()
	Apply the pending changes and commit.

	PendingUpdate (const PendingUpdate &)=delete

PendingUpdate &	operator= (const PendingUpdate &)=delete

	PendingUpdate (PendingUpdate &&) noexcept=default

PendingUpdate &	operator= (PendingUpdate &&) noexcept=default

Public Member Functions inherited from iceberg::ErrorCollector
	ErrorCollector (ErrorCollector &&)=default

ErrorCollector &	operator= (ErrorCollector &&)=default

	ErrorCollector (const ErrorCollector &)=default

ErrorCollector &	operator= (const ErrorCollector &)=default

template<typename... Args>
auto &	AddError (this auto &self, ErrorKind kind, const std::format_string< Args... > fmt, Args &&... args)
	Add a specific error and return reference to derived class.

auto &	AddError (this auto &self, Error err)
	Add an existing error object and return reference to derived class.

auto &	AddError (this auto &self, std::unexpected< Error > err)
	Add an unexpected result's error and return reference to derived class.

bool	has_errors () const
	Check if any errors have been collected.

size_t	error_count () const
	Get the number of errors collected.

Status	CheckErrors () const
	Check for accumulated errors and return them if any exist.

void	ClearErrors ()
	Clear all accumulated errors.

const std::vector< Error > &	errors () const
	Get read-only access to all collected errors.

Static Public Member Functions
static Result< std::unique_ptr< FastAppend > >	Make (std::string table_name, std::shared_ptr< TransactionContext > ctx)
	Create a new FastAppend instance.

Additional Inherited Members
Public Types inherited from iceberg::PendingUpdate
enum class	Kind : uint8_t { kExpireSnapshots , kSetSnapshot , kUpdateLocation , kUpdatePartitionSpec , kUpdatePartitionStatistics , kUpdateProperties , kUpdateSchema , kUpdateSnapshot , kUpdateSnapshotReference , kUpdateSortOrder , kUpdateStatistics }

Protected Member Functions inherited from iceberg::SnapshotUpdate
	SnapshotUpdate (std::shared_ptr< TransactionContext > ctx)

Result< std::vector< ManifestFile > >	WriteDataManifests (std::span< const std::shared_ptr< DataFile > > files, const std::shared_ptr< PartitionSpec > &spec, std::optional< int64_t > data_sequence_number=std::nullopt)
	Write data manifests for the given data files.

Result< std::vector< ManifestFile > >	WriteDeleteManifests (std::span< const ContentFileWithSequenceNumber > files, const std::shared_ptr< PartitionSpec > &spec)

const std::string &	target_branch () const

bool	can_inherit_snapshot_id () const

const std::string &	commit_uuid () const

int32_t	manifest_count () const

int32_t	attempt () const

int64_t	target_manifest_size_bytes () const

virtual Status	Validate (const TableMetadata &current_metadata, const std::shared_ptr< Snapshot > &snapshot)
	Validate the current metadata.

int64_t	SnapshotId ()
	Get or generate the snapshot ID for the new snapshot.

Status	DeleteFile (const std::string &path)
	Delete a file at the given path.

std::string	ManifestPath ()

std::string	ManifestListPath ()

SnapshotSummaryBuilder &	summary_builder ()

SnapshotSummaryBuilder	BuildManifestCountSummary (std::span< const ManifestFile > manifests, int32_t replaced_manifests_count)

Protected Member Functions inherited from iceberg::PendingUpdate
	PendingUpdate (std::shared_ptr< TransactionContext > ctx)

const TableMetadata &	base () const

Protected Attributes inherited from iceberg::SnapshotUpdate
SnapshotSummaryBuilder	summary_

Protected Attributes inherited from iceberg::PendingUpdate
std::shared_ptr< TransactionContext >	ctx_

Protected Attributes inherited from iceberg::ErrorCollector
std::vector< Error >	errors_

Detailed Description

API for appending new files in a table.

This API accumulates file additions, produces a new Snapshot of the table, and commits that snapshot as current. When committing, these changes are applied to the latest table snapshot. Commit conflicts are resolved by applying the changes to the new latest snapshot and reattempting the commit.

FastAppend is optimized for appending new data files to a table. It creates new manifest files for the added data without compacting or rewriting existing manifests.

Member Function Documentation

◆ AppendFile()

FastAppend & iceberg::FastAppend::AppendFile ( const std::shared_ptr< DataFile > & file )

Append a DataFile to the table.

Parameters

file	A data file.

Returns: This FastAppend for method chaining.

◆ AppendManifest()

FastAppend & iceberg::FastAppend::AppendManifest ( const ManifestFile & manifest )

Append a ManifestFile to the table.

The manifest must contain only appended files. All files in the manifest are appended to the table in the snapshot created by this update.

If the manifest doesn't have a snapshot ID assigned and snapshot ID inheritance is enabled, it will be used directly. Otherwise, it will be copied with the new snapshot ID.

Parameters

manifest A manifest file of files to append.

Returns: This FastAppend for method chaining.

◆ Apply()

Result< std::vector< ManifestFile > > iceberg::FastAppend::Apply	(	const TableMetadata &	metadata_to_update,
		const std::shared_ptr< Snapshot > &	snapshot
	)

overridevirtual

Apply the update's changes to the given metadata and snapshot.

Parameters

metadata_to_update	The base table metadata to apply changes to
snapshot	Snapshot to apply the changes to

Returns: A vector of manifest files for the new snapshot

Implements iceberg::SnapshotUpdate.

◆ CleanUncommitted()

Status iceberg::FastAppend::CleanUncommitted ( const std::unordered_set< std::string > & committed )

overridevirtual

Clean up any uncommitted manifests that were created.

Manifests may not be committed if Apply is called multiple times because a commit conflict has occurred. Implementations may keep around manifests because the same changes will be made by both Apply calls. This method instructs the implementation to clean up those manifests and passes the paths of the manifests that were actually committed.

Parameters

committed A set of manifest paths that were actually committed

Implements iceberg::SnapshotUpdate.

◆ CleanupAfterCommit()

bool iceberg::FastAppend::CleanupAfterCommit ( ) const

overridevirtual

Check if cleanup should happen after commit.

Returns: True if cleanup should happen after commit

Reimplemented from iceberg::SnapshotUpdate.

◆ Make()

Result< std::unique_ptr< FastAppend > > iceberg::FastAppend::Make	(	std::string	table_name,
		std::shared_ptr< TransactionContext >	ctx
	)

static

Create a new FastAppend instance.

Parameters

table_name	The name of the table
ctx	The transaction context to use for this update

Returns: A Result containing the FastAppend instance or an error

◆ operation()

std::string iceberg::FastAppend::operation ( )

overridevirtual

A string that describes the action that produced the new snapshot.

Returns: A string operation name

Implements iceberg::SnapshotUpdate.

◆ SetSummaryProperty()

void iceberg::FastAppend::SetSummaryProperty	(	const std::string &	property,
		const std::string &	value
	)

overridevirtual

Set a summary property.

Implementations may override this to retain custom properties across retry-safe summary rebuilds.

Reimplemented from iceberg::SnapshotUpdate.

◆ Summary()

std::unordered_map< std::string, std::string > iceberg::FastAppend::Summary ( )

overridevirtual

Get the summary map for this operation.

Returns: A map of summary properties

Implements iceberg::SnapshotUpdate.

The documentation for this class was generated from the following files:

iceberg/update/fast_append.h
iceberg/update/fast_append.cc

Public Member Functions

Static Public Member Functions

Additional Inherited Members

Detailed Description

Member Function Documentation

◆ AppendFile()

◆ AppendManifest()

◆ Apply()

◆ CleanUncommitted()

◆ CleanupAfterCommit()

◆ Make()

◆ operation()

◆ SetSummaryProperty()

◆ Summary()