table_metadata.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <memory>
#include <string>
#include <string_view>
#include <unordered_map>
#include <vector>
 
#include "iceberg/encryption/encrypted_key.h"
#include "iceberg/iceberg_export.h"
#include "iceberg/table_properties.h"
#include "iceberg/type_fwd.h"
#include "iceberg/util/error_collector.h"
#include "iceberg/util/lazy.h"
#include "iceberg/util/timepoint.h"
 
namespace iceberg {
 
struct ICEBERG_EXPORT SnapshotLogEntry {
  TimePointMs timestamp_ms;
  int64_t snapshot_id;
 
  friend bool operator==(const SnapshotLogEntry& lhs, const SnapshotLogEntry& rhs) {
    return lhs.timestamp_ms == rhs.timestamp_ms && lhs.snapshot_id == rhs.snapshot_id;
  }
};
 
struct ICEBERG_EXPORT MetadataLogEntry {
  TimePointMs timestamp_ms;
  std::string metadata_file;
 
  friend bool operator==(const MetadataLogEntry& lhs, const MetadataLogEntry& rhs) {
    return lhs.timestamp_ms == rhs.timestamp_ms && lhs.metadata_file == rhs.metadata_file;
  }
};
 
struct ICEBERG_EXPORT TableMetadata {
  static constexpr int8_t kDefaultTableFormatVersion = 2;
  static constexpr int8_t kSupportedTableFormatVersion = 3;
  static constexpr int8_t kMinFormatVersionRowLineage = 3;
  static constexpr int8_t kMinFormatVersionDefaultValues = 3;
  static constexpr int64_t kInitialSequenceNumber = 0;
  static constexpr int64_t kInitialRowId = 0;
 
  static inline const std::unordered_map<TypeId, int8_t> kMinFormatVersions = {
      {TypeId::kTimestampNs, 3}, {TypeId::kTimestampTzNs, 3}, {TypeId::kUnknown, 3},
      {TypeId::kVariant, 3},     {TypeId::kGeometry, 3},      {TypeId::kGeography, 3},
  };
 
  int8_t format_version;
  std::string table_uuid;
  std::string location;
  int64_t last_sequence_number;
  TimePointMs last_updated_ms;
  int32_t last_column_id;
  std::vector<std::shared_ptr<iceberg::Schema>> schemas;
  int32_t current_schema_id;
  std::vector<std::shared_ptr<iceberg::PartitionSpec>> partition_specs;
  int32_t default_spec_id;
  int32_t last_partition_id;
  TableProperties properties;
  int64_t current_snapshot_id;
  std::vector<std::shared_ptr<iceberg::Snapshot>> snapshots;
  std::vector<SnapshotLogEntry> snapshot_log;
  std::vector<MetadataLogEntry> metadata_log;
  std::vector<std::shared_ptr<iceberg::SortOrder>> sort_orders;
  int32_t default_sort_order_id;
  std::unordered_map<std::string, std::shared_ptr<SnapshotRef>> refs;
  std::vector<std::shared_ptr<struct StatisticsFile>> statistics;
  std::vector<std::shared_ptr<struct PartitionStatisticsFile>> partition_statistics;
  int64_t next_row_id;
  std::vector<EncryptedKey> encryption_keys;
 
  static Result<std::unique_ptr<TableMetadata>> Make(
      const iceberg::Schema& schema, const iceberg::PartitionSpec& spec,
      const iceberg::SortOrder& sort_order, const std::string& location,
      const std::unordered_map<std::string, std::string>& properties,
      int format_version = kDefaultTableFormatVersion);
 
  Result<std::shared_ptr<iceberg::Schema>> Schema() const;
  Result<std::shared_ptr<iceberg::Schema>> SchemaById(int32_t schema_id) const;
  Result<std::shared_ptr<iceberg::PartitionSpec>> PartitionSpec() const;
  Result<std::shared_ptr<iceberg::PartitionSpec>> PartitionSpecById(
      int32_t spec_id) const;
  Result<std::shared_ptr<iceberg::SortOrder>> SortOrder() const;
  Result<std::shared_ptr<iceberg::SortOrder>> SortOrderById(int32_t sort_order_id) const;
  Result<std::shared_ptr<iceberg::Snapshot>> Snapshot() const;
  Result<std::shared_ptr<iceberg::Snapshot>> SnapshotById(int64_t snapshot_id) const;
  int64_t NextSequenceNumber() const;
 
  ICEBERG_EXPORT friend bool operator==(const TableMetadata& lhs,
                                        const TableMetadata& rhs);
};
 
// Cache for table metadata mappings to facilitate fast lookups.
class ICEBERG_EXPORT TableMetadataCache {
 public:
  explicit TableMetadataCache(const TableMetadata* metadata) : metadata_(metadata) {}
 
  template <typename T>
  using ByIdMap = std::unordered_map<int32_t, std::shared_ptr<T>>;
  using SchemasMap = ByIdMap<Schema>;
  using PartitionSpecsMap = ByIdMap<PartitionSpec>;
  using SortOrdersMap = ByIdMap<SortOrder>;
  using SnapshotsMap = std::unordered_map<int64_t, std::shared_ptr<Snapshot>>;
  using SchemasMapRef = std::reference_wrapper<const SchemasMap>;
  using PartitionSpecsMapRef = std::reference_wrapper<const PartitionSpecsMap>;
  using SortOrdersMapRef = std::reference_wrapper<const SortOrdersMap>;
  using SnapshotsMapRef = std::reference_wrapper<const SnapshotsMap>;
 
  Result<SchemasMapRef> GetSchemasById() const;
  Result<PartitionSpecsMapRef> GetPartitionSpecsById() const;
  Result<SortOrdersMapRef> GetSortOrdersById() const;
  Result<SnapshotsMapRef> GetSnapshotsById() const;
 
 private:
  static Result<SchemasMap> InitSchemasMap(const TableMetadata* metadata);
  static Result<PartitionSpecsMap> InitPartitionSpecsMap(const TableMetadata* metadata);
  static Result<SortOrdersMap> InitSortOrdersMap(const TableMetadata* metadata);
  static Result<SnapshotsMap> InitSnapshotMap(const TableMetadata* metadata);
 
  const TableMetadata* metadata_;
  Lazy<InitSchemasMap> schemas_map_;
  Lazy<InitPartitionSpecsMap> partition_specs_map_;
  Lazy<InitSortOrdersMap> sort_orders_map_;
  Lazy<InitSnapshotMap> snapshot_map_;
};
 
ICEBERG_EXPORT std::string ToString(const SnapshotLogEntry& entry);
 
ICEBERG_EXPORT std::string ToString(const MetadataLogEntry& entry);
 
class ICEBERG_EXPORT TableMetadataBuilder : public ErrorCollector {
 public:
  static std::unique_ptr<TableMetadataBuilder> BuildFromEmpty(
      int8_t format_version = TableMetadata::kDefaultTableFormatVersion);
 
  static std::unique_ptr<TableMetadataBuilder> BuildFrom(const TableMetadata* base);
 
  TableMetadataBuilder& ApplyChangesForCreate(const TableMetadata& base);
 
  TableMetadataBuilder& SetMetadataLocation(std::string_view metadata_location);
 
  TableMetadataBuilder& SetPreviousMetadataLocation(
      std::string_view previous_metadata_location);
 
  TableMetadataBuilder& AssignUUID();
 
  TableMetadataBuilder& AssignUUID(std::string_view uuid);
 
  TableMetadataBuilder& UpgradeFormatVersion(int8_t new_format_version);
 
  TableMetadataBuilder& SetCurrentSchema(const std::shared_ptr<Schema>& schema,
                                         int32_t new_last_column_id);
 
  TableMetadataBuilder& SetCurrentSchema(int32_t schema_id);
 
  TableMetadataBuilder& AddSchema(const std::shared_ptr<Schema>& schema);
 
  TableMetadataBuilder& SetDefaultPartitionSpec(std::shared_ptr<PartitionSpec> spec);
 
  TableMetadataBuilder& SetDefaultPartitionSpec(int32_t spec_id);
 
  TableMetadataBuilder& AddPartitionSpec(std::shared_ptr<PartitionSpec> spec);
 
  TableMetadataBuilder& RemovePartitionSpecs(const std::vector<int32_t>& spec_ids);
 
  TableMetadataBuilder& RemoveSchemas(const std::unordered_set<int32_t>& schema_ids);
 
  TableMetadataBuilder& SetDefaultSortOrder(std::shared_ptr<SortOrder> order);
 
  TableMetadataBuilder& SetDefaultSortOrder(int32_t order_id);
 
  TableMetadataBuilder& AddSortOrder(std::shared_ptr<SortOrder> order);
 
  TableMetadataBuilder& AddSnapshot(std::shared_ptr<Snapshot> snapshot);
 
  TableMetadataBuilder& SetBranchSnapshot(int64_t snapshot_id, const std::string& branch);
 
  TableMetadataBuilder& SetBranchSnapshot(std::shared_ptr<Snapshot> snapshot,
                                          const std::string& branch);
 
  TableMetadataBuilder& SetRef(const std::string& name, std::shared_ptr<SnapshotRef> ref);
 
  TableMetadataBuilder& RemoveRef(const std::string& name);
 
  TableMetadataBuilder& RemoveSnapshots(
      const std::vector<std::shared_ptr<Snapshot>>& snapshots_to_remove);
 
  TableMetadataBuilder& RemoveSnapshots(const std::vector<int64_t>& snapshot_ids);
 
  TableMetadataBuilder& SuppressHistoricalSnapshots();
 
  TableMetadataBuilder& SetStatistics(std::shared_ptr<StatisticsFile> statistics_file);
 
  TableMetadataBuilder& RemoveStatistics(int64_t snapshot_id);
 
  TableMetadataBuilder& SetPartitionStatistics(
      const std::shared_ptr<PartitionStatisticsFile>& partition_statistics_file);
 
  TableMetadataBuilder& RemovePartitionStatistics(int64_t snapshot_id);
 
  TableMetadataBuilder& SetProperties(
      const std::unordered_map<std::string, std::string>& updated);
 
  TableMetadataBuilder& RemoveProperties(const std::unordered_set<std::string>& removed);
 
  TableMetadataBuilder& SetLocation(std::string_view location);
 
  TableMetadataBuilder& AddEncryptionKey(EncryptedKey key);
 
  TableMetadataBuilder& RemoveEncryptionKey(std::string_view key_id);
 
  Result<std::unique_ptr<TableMetadata>> Build();
 
  const std::vector<std::unique_ptr<TableUpdate>>& changes() const;
 
  const TableMetadata* base() const;
 
  const TableMetadata& current() const;
 
  ~TableMetadataBuilder() override;
 
  // Delete copy operations (use BuildFrom to create a new builder)
  TableMetadataBuilder(const TableMetadataBuilder&) = delete;
  TableMetadataBuilder& operator=(const TableMetadataBuilder&) = delete;
 
  // Enable move operations
  TableMetadataBuilder(TableMetadataBuilder&&) noexcept;
  TableMetadataBuilder& operator=(TableMetadataBuilder&&) noexcept;
 
 private:
  explicit TableMetadataBuilder(int8_t format_version);
 
  explicit TableMetadataBuilder(const TableMetadata* base);
 
  struct Impl;
  std::unique_ptr<Impl> impl_;
};
 
enum class ICEBERG_EXPORT MetadataFileCodecType {
  kNone,
  kGzip,
};
 
struct ICEBERG_EXPORT TableMetadataUtil {
  struct ICEBERG_EXPORT Codec {
    static Result<MetadataFileCodecType> FromString(std::string_view name);
 
    static Result<MetadataFileCodecType> FromFileName(std::string_view file_name);
 
    static Result<std::string> NameToFileExtension(std::string_view codec);
 
    static std::string TypeToFileExtension(MetadataFileCodecType codec);
 
    static constexpr std::string_view kTableMetadataFileSuffix = ".metadata.json";
    static constexpr std::string_view kCompGzipTableMetadataFileSuffix =
        ".metadata.json.gz";
    static constexpr std::string_view kGzipTableMetadataFileSuffix = ".gz.metadata.json";
    static constexpr std::string_view kGzipTableMetadataFileExtension = ".gz";
    static constexpr std::string_view kCodecTypeGzip = "GZIP";
    static constexpr std::string_view kCodecTypeNone = "NONE";
  };
 
  static Result<std::unique_ptr<TableMetadata>> Read(
      class FileIO& io, const std::string& location,
      std::optional<size_t> length = std::nullopt);
 
  static Result<std::string> Write(FileIO& io, const TableMetadata* base,
                                   const std::string& base_metadata_location,
                                   TableMetadata& metadata);
 
  static void DeleteRemovedMetadataFiles(FileIO& io, const TableMetadata* base,
                                         const TableMetadata& metadata);
 
  static Status Write(FileIO& io, const std::string& location,
                      const TableMetadata& metadata);
 
 private:
  static int32_t ParseVersionFromLocation(std::string_view metadata_location);
 
  static Result<std::string> NewTableMetadataFilePath(const TableMetadata& metadata,
                                                      int32_t version);
};
 
}  // namespace iceberg
 
namespace std {
template <>
struct hash<iceberg::MetadataLogEntry> {
  size_t operator()(const iceberg::MetadataLogEntry& m) const noexcept {
    return std::hash<std::string>{}(m.metadata_file);
  }
};
 
}  // namespace std