manifest_list.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
 
#include "iceberg/compat/unreachable.h"
#include "iceberg/constants.h"
#include "iceberg/iceberg_export.h"
#include "iceberg/partition_spec.h"
#include "iceberg/result.h"
#include "iceberg/schema_field.h"
#include "iceberg/table_metadata.h"
#include "iceberg/type.h"
 
namespace iceberg {
 
struct ICEBERG_EXPORT PartitionFieldSummary {
  bool contains_null = true;
  std::optional<bool> contains_nan;
  std::optional<std::vector<uint8_t>> lower_bound;
  std::optional<std::vector<uint8_t>> upper_bound;
 
  inline static const SchemaField kContainsNull =
      SchemaField::MakeRequired(509, "contains_null", iceberg::boolean(),
                                "True if any file has a null partition value");
  inline static const SchemaField kContainsNaN =
      SchemaField::MakeOptional(518, "contains_nan", iceberg::boolean(),
                                "True if any file has a nan partition value");
  inline static const SchemaField kLowerBound = SchemaField::MakeOptional(
      510, "lower_bound", iceberg::binary(), "Partition lower bound for all files");
  inline static const SchemaField kUpperBound = SchemaField::MakeOptional(
      511, "upper_bound", iceberg::binary(), "Partition upper bound for all files");
 
  bool operator==(const PartitionFieldSummary& other) const = default;
 
  static const std::shared_ptr<StructType>& Type();
};
 
enum class ManifestContent {
  kData = 0,
  kDeletes = 1,
};
 
struct ICEBERG_EXPORT ManifestFile {
  std::string manifest_path;
  int64_t manifest_length = 0;
  int32_t partition_spec_id = PartitionSpec::kInitialSpecId;
  ManifestContent content = ManifestContent::kData;
  int64_t sequence_number = TableMetadata::kInitialSequenceNumber;
  int64_t min_sequence_number = TableMetadata::kInitialSequenceNumber;
  std::optional<int64_t> added_snapshot_id;
  std::optional<int32_t> added_files_count;
  std::optional<int32_t> existing_files_count;
  std::optional<int32_t> deleted_files_count;
  std::optional<int64_t> added_rows_count;
  std::optional<int64_t> existing_rows_count;
  std::optional<int64_t> deleted_rows_count;
  std::vector<PartitionFieldSummary> partitions;
  std::vector<uint8_t> key_metadata;
  std::optional<int64_t> first_row_id;
 
  bool has_added_files() const { return added_files_count.value_or(1) > 0; }
 
  bool has_existing_files() const { return existing_files_count.value_or(1) > 0; }
 
  bool has_deleted_files() const { return deleted_files_count.value_or(1) > 0; }
 
  static constexpr int32_t kManifestPathFieldId = 500;
  inline static const SchemaField kManifestPath = SchemaField::MakeRequired(
      kManifestPathFieldId, "manifest_path", string(), "Location URI with FS scheme");
 
  static constexpr int32_t kManifestLengthFieldId = 501;
  inline static const SchemaField kManifestLength = SchemaField::MakeRequired(
      kManifestLengthFieldId, "manifest_length", int64(), "Total file size in bytes");
 
  static constexpr int32_t kPartitionSpecIdFieldId = 502;
  inline static const SchemaField kPartitionSpecId = SchemaField::MakeRequired(
      kPartitionSpecIdFieldId, "partition_spec_id", int32(), "Spec ID used to write");
 
  static constexpr int32_t kContentFieldId = 517;
  inline static const SchemaField kContent = SchemaField::MakeOptional(
      kContentFieldId, "content", int32(), "Contents of the manifest: 0=data, 1=deletes");
 
  static constexpr int32_t kSequenceNumberFieldId = 515;
  inline static const SchemaField kSequenceNumber =
      SchemaField::MakeOptional(kSequenceNumberFieldId, "sequence_number", int64(),
                                "Sequence number when the manifest was added");
 
  static constexpr int32_t kMinSequenceNumberFieldId = 516;
  inline static const SchemaField kMinSequenceNumber =
      SchemaField::MakeOptional(kMinSequenceNumberFieldId, "min_sequence_number", int64(),
                                "Lowest sequence number in the manifest");
 
  static constexpr int32_t kAddedSnapshotIdFieldId = 503;
  inline static const SchemaField kAddedSnapshotId =
      SchemaField::MakeRequired(kAddedSnapshotIdFieldId, "added_snapshot_id", int64(),
                                "Snapshot ID that added the manifest");
 
  static constexpr int32_t kAddedFilesCountFieldId = 504;
  inline static const SchemaField kAddedFilesCount = SchemaField::MakeOptional(
      kAddedFilesCountFieldId, "added_files_count", int32(), "Added entry count");
 
  static constexpr int32_t kExistingFilesCountFieldId = 505;
  inline static const SchemaField kExistingFilesCount =
      SchemaField::MakeOptional(kExistingFilesCountFieldId, "existing_files_count",
                                int32(), "Existing entry count");
 
  static constexpr int32_t kDeletedFilesCountFieldId = 506;
  inline static const SchemaField kDeletedFilesCount = SchemaField::MakeOptional(
      kDeletedFilesCountFieldId, "deleted_files_count", int32(), "Deleted entry count");
 
  static constexpr int32_t kAddedRowsCountFieldId = 512;
  inline static const SchemaField kAddedRowsCount = SchemaField::MakeOptional(
      kAddedRowsCountFieldId, "added_rows_count", int64(), "Added rows count");
 
  static constexpr int32_t kExistingRowsCountFieldId = 513;
  inline static const SchemaField kExistingRowsCount = SchemaField::MakeOptional(
      kExistingRowsCountFieldId, "existing_rows_count", int64(), "Existing rows count");
 
  static constexpr int32_t kDeletedRowsCountFieldId = 514;
  inline static const SchemaField kDeletedRowsCount = SchemaField::MakeOptional(
      kDeletedRowsCountFieldId, "deleted_rows_count", int64(), "Deleted rows count");
 
  static constexpr int32_t kPartitionSummaryFieldId = 507;
  inline static const SchemaField kPartitions = SchemaField::MakeOptional(
      kPartitionSummaryFieldId, "partitions",
      list(SchemaField::MakeRequired(508, std::string(ListType::kElementName),
                                     struct_({
                                         PartitionFieldSummary::kContainsNull,
                                         PartitionFieldSummary::kContainsNaN,
                                         PartitionFieldSummary::kLowerBound,
                                         PartitionFieldSummary::kUpperBound,
                                     }))),
      "Summary for each partition");
 
  static constexpr int32_t kKeyMetadataFieldId = 519;
  inline static const SchemaField kKeyMetadata = SchemaField::MakeOptional(
      kKeyMetadataFieldId, "key_metadata", binary(), "Encryption key metadata blob");
 
  static constexpr int32_t kFirstRowIdFieldId = 520;
  inline static const SchemaField kFirstRowId = SchemaField::MakeOptional(
      kFirstRowIdFieldId, "first_row_id", int64(),
      "Starting row ID to assign to new rows in ADDED data files");
 
  bool operator==(const ManifestFile& other) const {
    return manifest_path == other.manifest_path;
  }
 
  static const std::shared_ptr<StructType>& Type();
};
 
struct ICEBERG_EXPORT ManifestList {
  std::vector<ManifestFile> entries;
};
 
ICEBERG_EXPORT inline constexpr std::string_view ToString(ManifestContent type) noexcept {
  switch (type) {
    case ManifestContent::kData:
      return "data";
    case ManifestContent::kDeletes:
      return "deletes";
  }
  ::iceberg::unreachable();
}
 
ICEBERG_EXPORT inline constexpr Result<ManifestContent> ManifestContentFromString(
    std::string_view str) noexcept {
  if (str == "data") return ManifestContent::kData;
  if (str == "deletes") return ManifestContent::kDeletes;
  return InvalidArgument("Invalid manifest content type: {}", str);
}
 
ICEBERG_EXPORT inline constexpr Result<ManifestContent> ManifestContentFromInt(
    int32_t content) noexcept {
  switch (content) {
    case 0:
      return ManifestContent::kData;
    case 1:
      return ManifestContent::kDeletes;
    default:
      return InvalidArgument("Invalid manifest content type: {}", content);
  }
}
 
}  // namespace iceberg
 
namespace std {
template <>
struct hash<iceberg::ManifestFile> {
  size_t operator()(const iceberg::ManifestFile& manifest_file) const {
    return std::hash<std::string>{}(manifest_file.manifest_path);
  }
};
}  // namespace std