API for rewriting manifests for a table. More...

#include <rewrite_manifests.h>

Inheritance diagram for iceberg::RewriteManifests:

Public Types
using	ClusterByFunc = std::function< std::string(const DataFile &)>

using	RewritePredicate = std::function< bool(const ManifestFile &)>

Public Types inherited from iceberg::PendingUpdate
enum class	Kind : uint8_t { kExpireSnapshots , kSetSnapshot , kUpdateLocation , kUpdatePartitionSpec , kUpdatePartitionStatistics , kUpdateProperties , kUpdateSchema , kUpdateSnapshot , kUpdateSnapshotReference , kUpdateSortOrder , kUpdateStatistics }

Public Member Functions
RewriteManifests &	ClusterBy (ClusterByFunc func)
	Groups an existing data file by a cluster key produced by a function.

RewriteManifests &	RewriteIf (RewritePredicate predicate)
	Determines which existing manifest files for the table should be rewritten.

RewriteManifests &	DeleteManifest (const ManifestFile &manifest)
	Deletes a manifest file from the table.

RewriteManifests &	AddManifest (const ManifestFile &manifest)
	Adds a manifest file to the table.

Public Member Functions inherited from iceberg::SnapshotUpdate
Kind	kind () const override
	Return the kind of this pending update.

bool	IsRetryable () const override
	Whether this update can be retried after a commit conflict.

Status	Commit () override
	Apply the pending changes and commit.

auto &	ReportWith (this auto &self, std::shared_ptr< MetricsReporter > reporter)
	Set the metrics reporter for this snapshot update.

auto &	DeleteWith (this auto &self, std::function< Status(const std::string &)> delete_func)
	Set a callback to delete files instead of the table's default.

auto &	StageOnly (this auto &self)
	Stage a snapshot in table metadata, but do not make it current.

auto &	ScanManifestsWith (this auto &self, Executor &executor)
	Configure an executor for manifest planning work.

auto &	ToBranch (this auto &self, const std::string &branch)
	Perform operations on a particular branch.

auto &	Set (this auto &self, const std::string &property, const std::string &value)
	Set a summary property in the snapshot produced by this update.

auto &	WriteManifestsWith (this auto &self, Executor &executor, int32_t parallelism)
	Configure an executor and max writer count for writing new manifests.

Public Member Functions inherited from iceberg::PendingUpdate
	PendingUpdate (const PendingUpdate &)=delete

PendingUpdate &	operator= (const PendingUpdate &)=delete

	PendingUpdate (PendingUpdate &&)=delete

PendingUpdate &	operator= (PendingUpdate &&)=delete

Public Member Functions inherited from iceberg::ErrorCollector
	ErrorCollector (ErrorCollector &&)=default

ErrorCollector &	operator= (ErrorCollector &&)=default

	ErrorCollector (const ErrorCollector &)=default

ErrorCollector &	operator= (const ErrorCollector &)=default

template<typename... Args>
auto &	AddError (this auto &self, ErrorKind kind, const std::format_string< Args... > fmt, Args &&... args)
	Add a specific error.

auto &	AddError (this auto &self, Error err)
	Add an existing error object.

auto &	AddError (this auto &self, ::iceberg::unexpected< Error > err)
	Add an unexpected result's error.

auto &	AddError (this auto &self, std::unexpected< Error > err)

bool	has_errors () const
	Check if any errors have been collected.

size_t	error_count () const
	Get the number of errors collected.

Status	CheckErrors () const
	Check for accumulated errors and return them if any exist.

void	ClearErrors ()
	Clear all accumulated errors.

const std::vector< Error > &	errors () const
	Get read-only access to all collected errors.

Static Public Member Functions
static Result< std::shared_ptr< RewriteManifests > >	Make (std::string table_name, std::shared_ptr< TransactionContext > ctx)

Protected Member Functions
std::string	operation () override
	A string that describes the action that produced the new snapshot.

Result< std::vector< ManifestFile > >	Apply (const TableMetadata &metadata_to_update, const std::shared_ptr< Snapshot > &snapshot) override
	Apply the update's changes to the given metadata and snapshot.

std::unordered_map< std::string, std::string >	Summary () override
	Get the summary map for this operation.

Status	CleanUncommitted (const std::unordered_set< std::string > &committed) override
	Clean up any uncommitted manifests that were created.

	RewriteManifests (std::string table_name, std::shared_ptr< TransactionContext > ctx)

Protected Member Functions inherited from iceberg::SnapshotUpdate
virtual Result< ApplyResult >	Apply ()
	Build snapshot state for the transaction's current metadata.

Status	Finalize (const TableMetadata &committed) override
	Complete a known successful commit. Called only by Transaction with the final committed table metadata.

Status	CleanStaged () override
	Discard this generation's staging, retaining applied operation intent.

	SnapshotUpdate (std::shared_ptr< TransactionContext > ctx)

Result< std::vector< ManifestFile > >	WriteDataManifests (std::span< const std::shared_ptr< DataFile > > files, const std::shared_ptr< PartitionSpec > &spec, std::optional< int64_t > data_sequence_number=std::nullopt)
	Write data manifests for the given data files.

Result< std::vector< ManifestFile > >	WriteDeleteManifests (std::span< const ContentFileWithSequenceNumber > files, const std::shared_ptr< PartitionSpec > &spec)

const std::string &	target_branch () const

bool	can_inherit_snapshot_id () const

const std::string &	commit_uuid () const

int32_t	manifest_count () const

int32_t	attempt () const

int64_t	target_manifest_size_bytes () const

virtual Status	Validate (const TableMetadata &current_metadata, const std::shared_ptr< Snapshot > &snapshot)
	Validate the current metadata.

virtual void	SetSummaryProperty (const std::string &property, const std::string &value)
	Set a summary property.

int64_t	SnapshotId ()
	Get or generate the snapshot ID for the new snapshot.

Status	DeleteFile (const std::string &path) noexcept
	Best-effort delete. Failed paths remain registered for later cleanup.

void	RegisterStagedFile (const std::string &path)

void	UnregisterStagedFile (const std::string &path)
	Transfer cleanup ownership to the derived update.

std::string	ManifestPath ()

std::string	ManifestListPath ()

OptionalExecutor	plan_executor () const

SnapshotSummaryBuilder &	summary_builder ()

SnapshotSummaryBuilder	BuildManifestCountSummary (std::span< const ManifestFile > manifests, int32_t replaced_manifests_count)

Protected Member Functions inherited from iceberg::PendingUpdate
	PendingUpdate (std::shared_ptr< TransactionContext > ctx)

const TableMetadata &	base () const

Status	CheckCommitAllowed () const

Additional Inherited Members
Protected Attributes inherited from iceberg::SnapshotUpdate
SnapshotSummaryBuilder	summary_

Protected Attributes inherited from iceberg::PendingUpdate
std::shared_ptr< TransactionContext >	ctx_

Protected Attributes inherited from iceberg::ErrorCollector
std::vector< Error >	errors_

Detailed Description

API for rewriting manifests for a table.

This API accumulates manifest files, produces a new snapshot of the table described only by the manifest files that were added, and commits that snapshot as the current.

This API can be used to rewrite matching manifests according to a clustering function as well as to replace specific manifests. Manifests that are deleted or added directly are ignored during the rewrite process. The set of active files in replaced manifests must be the same as in new manifests.

When committing, these changes will be applied to the latest table snapshot. Commit conflicts will be resolved by applying the changes to the new latest snapshot and reattempting the commit.

Member Function Documentation

◆ AddManifest()

RewriteManifests & iceberg::RewriteManifests::AddManifest ( const ManifestFile & manifest )

Adds a manifest file to the table.

The added manifest cannot contain new or deleted files.

By default, the manifest is rewritten to ensure all entries have explicit snapshot IDs. In that case, the caller manages the original manifest and this update manages the copy.

If manifest entries are allowed to inherit the snapshot ID assigned on commit, the original manifest becomes part of the table metadata when the commit succeeds and must not be deleted manually. If the manifest gets merged while preparing a new snapshot, it is deleted automatically when this operation succeeds. If the commit fails, the original manifest is retained and may be deleted or reused by the caller.

◆ Apply()

Result< std::vector< ManifestFile > > iceberg::RewriteManifests::Apply	(	const TableMetadata &	metadata_to_update,
		const std::shared_ptr< Snapshot > &	snapshot
	)

overrideprotectedvirtual

Apply the update's changes to the given metadata and snapshot.

Parameters

metadata_to_update	The base table metadata to apply changes to
snapshot	Snapshot to apply the changes to

Returns: A vector of manifest files for the new snapshot

Implements iceberg::SnapshotUpdate.

◆ CleanUncommitted()

Status iceberg::RewriteManifests::CleanUncommitted ( const std::unordered_set< std::string > & committed )

overrideprotectedvirtual

Clean up any uncommitted manifests that were created.

Manifests may not be committed if Apply is called multiple times because a commit conflict has occurred. Implementations may keep around manifests because the same changes will be made by both Apply calls. This method instructs the implementation to clean up those manifests and passes the paths of the manifests that were actually committed.

Parameters

committed A set of manifest paths that were actually committed

Implements iceberg::SnapshotUpdate.

◆ ClusterBy()

RewriteManifests & iceberg::RewriteManifests::ClusterBy ( ClusterByFunc func )

Groups an existing data file by a cluster key produced by a function.

The cluster key determines which data file will be associated with a particular manifest. All data files with the same cluster key will be written to the same manifest unless the file is large and split into multiple files. Manifests deleted via DeleteManifest or added via AddManifest are ignored during the rewrite process.

◆ operation()

std::string iceberg::RewriteManifests::operation ( )

overrideprotectedvirtual

A string that describes the action that produced the new snapshot.

Returns: A string operation name

Implements iceberg::SnapshotUpdate.

◆ RewriteIf()

RewriteManifests & iceberg::RewriteManifests::RewriteIf ( RewritePredicate predicate )

Determines which existing manifest files for the table should be rewritten.

Manifests that do not match the predicate are kept as-is. If this is not called and no predicate is set, all manifests are rewritten.

◆ Summary()

std::unordered_map< std::string, std::string > iceberg::RewriteManifests::Summary ( )

overrideprotectedvirtual

Get the summary map for this operation.

Returns: A map of summary properties

Implements iceberg::SnapshotUpdate.

The documentation for this class was generated from the following file:

iceberg/update/rewrite_manifests.h

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Member Function Documentation

◆ AddManifest()

◆ Apply()

◆ CleanUncommitted()

◆ ClusterBy()

◆ operation()

◆ RewriteIf()

◆ Summary()