manifest_entry.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <cstdint>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <vector>
 
#include "iceberg/file_format.h"
#include "iceberg/iceberg_export.h"
#include "iceberg/result.h"
#include "iceberg/row/partition_values.h"
#include "iceberg/schema_field.h"
#include "iceberg/type.h"
 
namespace iceberg {
 
enum class ManifestStatus {
  kExisting = 0,
  kAdded = 1,
  kDeleted = 2,
};
 
ICEBERG_EXPORT constexpr Result<ManifestStatus> ManifestStatusFromInt(
    int32_t status) noexcept {
  switch (status) {
    case 0:
      return ManifestStatus::kExisting;
    case 1:
      return ManifestStatus::kAdded;
    case 2:
      return ManifestStatus::kDeleted;
    default:
      return InvalidArgument("Invalid manifest status: {}", status);
  }
}
 
struct ICEBERG_EXPORT DataFile {
  enum class Content {
    kData = 0,
    kPositionDeletes = 1,
    kEqualityDeletes = 2,
  };
 
  Content content = Content::kData;
  std::string file_path;
  FileFormatType file_format = FileFormatType::kParquet;
  PartitionValues partition;
  int64_t record_count = 0;
  int64_t file_size_in_bytes = 0;
  std::map<int32_t, int64_t> column_sizes;
  std::map<int32_t, int64_t> value_counts;
  std::map<int32_t, int64_t> null_value_counts;
  std::map<int32_t, int64_t> nan_value_counts;
  std::map<int32_t, std::vector<uint8_t>> lower_bounds;
  std::map<int32_t, std::vector<uint8_t>> upper_bounds;
  std::vector<uint8_t> key_metadata;
  std::vector<int64_t> split_offsets;
  std::vector<int32_t> equality_ids;
  std::optional<int32_t> sort_order_id;
  std::optional<int64_t> first_row_id;
  std::optional<std::string> referenced_data_file;
  std::optional<int64_t> content_offset;
  std::optional<int64_t> content_size_in_bytes;
 
 
  std::optional<int32_t> partition_spec_id;
 
  std::optional<int64_t> data_sequence_number;
 
  static constexpr int32_t kContentFieldId = 134;
  inline static const SchemaField kContent = SchemaField::MakeOptional(
      kContentFieldId, "content", int32(),
      "Contents of the file: 0=data, 1=position deletes, 2=equality deletes");
 
  static constexpr int32_t kFilePathFieldId = 100;
  inline static const SchemaField kFilePath = SchemaField::MakeRequired(
      kFilePathFieldId, "file_path", string(), "Location URI with FS scheme");
 
  static constexpr int32_t kFileFormatFieldId = 101;
  inline static const SchemaField kFileFormat =
      SchemaField::MakeRequired(kFileFormatFieldId, "file_format", string(),
                                "File format name: avro, orc, or parquet");
 
  static constexpr int32_t kSpecIdFieldId = 141;
  inline static const SchemaField kSpecId =
      SchemaField::MakeOptional(kSpecIdFieldId, "spec_id", int32(), "Partition spec ID");
 
  static constexpr int32_t kPartitionFieldId = 102;
  inline static const std::string kPartitionField = "partition";
  inline static const std::string kPartitionDoc =
      "Partition data tuple, schema based on the partition spec";
 
  static constexpr int32_t kRecordCountFieldId = 103;
  inline static const SchemaField kRecordCount = SchemaField::MakeRequired(
      kRecordCountFieldId, "record_count", int64(), "Number of records in the file");
 
  static constexpr int32_t kFileSizeFieldId = 104;
  inline static const SchemaField kFileSize = SchemaField::MakeRequired(
      kFileSizeFieldId, "file_size_in_bytes", int64(), "Total file size in bytes");
 
  static constexpr int32_t kColumnSizesFieldId = 108;
  inline static const SchemaField kColumnSizes = SchemaField::MakeOptional(
      kColumnSizesFieldId, "column_sizes",
      map(SchemaField::MakeRequired(117, std::string(MapType::kKeyName), int32()),
          SchemaField::MakeRequired(118, std::string(MapType::kValueName), int64())),
      "Map of column id to total size on disk");
 
  static constexpr int32_t kValueCountsFieldId = 109;
  inline static const SchemaField kValueCounts = SchemaField::MakeOptional(
      kValueCountsFieldId, "value_counts",
      map(SchemaField::MakeRequired(119, std::string(MapType::kKeyName), int32()),
          SchemaField::MakeRequired(120, std::string(MapType::kValueName), int64())),
      "Map of column id to total count, including null and NaN");
 
  static constexpr int32_t kNullValueCountsFieldId = 110;
  inline static const SchemaField kNullValueCounts = SchemaField::MakeOptional(
      kNullValueCountsFieldId, "null_value_counts",
      map(SchemaField::MakeRequired(121, std::string(MapType::kKeyName), int32()),
          SchemaField::MakeRequired(122, std::string(MapType::kValueName), int64())),
      "Map of column id to null value count");
 
  static constexpr int32_t kNanValueCountsFieldId = 137;
  inline static const SchemaField kNanValueCounts = SchemaField::MakeOptional(
      kNanValueCountsFieldId, "nan_value_counts",
      map(SchemaField::MakeRequired(138, std::string(MapType::kKeyName), int32()),
          SchemaField::MakeRequired(139, std::string(MapType::kValueName), int64())),
      "Map of column id to number of NaN values in the column");
 
  static constexpr int32_t kLowerBoundsFieldId = 125;
  inline static const SchemaField kLowerBounds = SchemaField::MakeOptional(
      kLowerBoundsFieldId, "lower_bounds",
      map(SchemaField::MakeRequired(126, std::string(MapType::kKeyName), int32()),
          SchemaField::MakeRequired(127, std::string(MapType::kValueName), binary())),
      "Map of column id to lower bound");
 
  static constexpr int32_t kUpperBoundsFieldId = 128;
  inline static const SchemaField kUpperBounds = SchemaField::MakeOptional(
      kUpperBoundsFieldId, "upper_bounds",
      map(SchemaField::MakeRequired(129, std::string(MapType::kKeyName), int32()),
          SchemaField::MakeRequired(130, std::string(MapType::kValueName), binary())),
      "Map of column id to upper bound");
 
  static constexpr int32_t kKeyMetadataFieldId = 131;
  inline static const SchemaField kKeyMetadata = SchemaField::MakeOptional(
      kKeyMetadataFieldId, "key_metadata", binary(), "Encryption key metadata blob");
 
  static constexpr int32_t kSplitOffsetsFieldId = 132;
  inline static const SchemaField kSplitOffsets = SchemaField::MakeOptional(
      kSplitOffsetsFieldId, "split_offsets",
      list(SchemaField::MakeRequired(133, std::string(ListType::kElementName), int64())),
      "Splittable offsets");
 
  static constexpr int32_t kEqualityIdsFieldId = 135;
  inline static const SchemaField kEqualityIds = SchemaField::MakeOptional(
      kEqualityIdsFieldId, "equality_ids",
      list(SchemaField::MakeRequired(136, std::string(ListType::kElementName), int32())),
      "Equality comparison field IDs");
 
  static constexpr int32_t kSortOrderIdFieldId = 140;
  inline static const SchemaField kSortOrderId = SchemaField::MakeOptional(
      kSortOrderIdFieldId, "sort_order_id", int32(), "Sort order ID");
 
  static constexpr int32_t kFirstRowIdFieldId = 142;
  inline static const SchemaField kFirstRowId =
      SchemaField::MakeOptional(kFirstRowIdFieldId, "first_row_id", int64(),
                                "Starting row ID to assign to new rows");
 
  static constexpr int32_t kReferencedDataFileFieldId = 143;
  inline static const SchemaField kReferencedDataFile = SchemaField::MakeOptional(
      kReferencedDataFileFieldId, "referenced_data_file", string(),
      "Fully qualified location (URI with FS scheme) of a data file that all deletes "
      "reference");
 
  static constexpr int32_t kContentOffsetFieldId = 144;
  inline static const SchemaField kContentOffset =
      SchemaField::MakeOptional(kContentOffsetFieldId, "content_offset", int64(),
                                "The offset in the file where the content starts");
 
  static constexpr int32_t kContentSizeFieldId = 145;
  inline static const SchemaField kContentSize =
      SchemaField::MakeOptional(kContentSizeFieldId, "content_size_in_bytes", int64(),
                                "The length of referenced content stored in the file");
 
  bool operator==(const DataFile& other) const = default;
 
  static std::shared_ptr<StructType> Type(std::shared_ptr<StructType> partition_type);
 
  bool IsDeletionVector() const {
    return content == Content::kPositionDeletes && file_format == FileFormatType::kPuffin;
  }
};
 
 
struct ICEBERG_EXPORT ManifestEntry {
  ManifestStatus status = ManifestStatus::kAdded;
  std::optional<int64_t> snapshot_id;
  std::optional<int64_t> sequence_number;
  std::optional<int64_t> file_sequence_number;
  std::shared_ptr<DataFile> data_file;
 
  static constexpr int32_t kStatusFieldId = 0;
  inline static const SchemaField kStatus =
      SchemaField::MakeRequired(kStatusFieldId, "status", int32());
 
  static constexpr int32_t kSnapshotIdFieldId = 1;
  inline static const SchemaField kSnapshotId =
      SchemaField::MakeOptional(kSnapshotIdFieldId, "snapshot_id", int64());
 
  static constexpr int32_t kDataFileFieldId = 2;
  inline static const std::string kDataFileField = "data_file";
 
  static constexpr int32_t kSequenceNumberFieldId = 3;
  inline static const SchemaField kSequenceNumber =
      SchemaField::MakeOptional(kSequenceNumberFieldId, "sequence_number", int64());
 
  static constexpr int32_t kFileSequenceNumberFieldId = 4;
  inline static const SchemaField kFileSequenceNumber = SchemaField::MakeOptional(
      kFileSequenceNumberFieldId, "file_sequence_number", int64());
 
  constexpr bool IsAlive() const {
    return status == ManifestStatus::kAdded || status == ManifestStatus::kExisting;
  }
 
  ManifestEntry AsAdded() const {
    ManifestEntry copy = *this;
    copy.status = ManifestStatus::kAdded;
    if (copy.data_file != nullptr && copy.data_file->first_row_id.has_value()) {
      copy.data_file = std::make_unique<DataFile>(*copy.data_file);
      copy.data_file->first_row_id = std::nullopt;
    }
    return copy;
  }
 
  ManifestEntry AsExisting() const {
    ManifestEntry copy = *this;
    copy.status = ManifestStatus::kExisting;
    return copy;
  }
 
  ManifestEntry AsDeleted() const {
    ManifestEntry copy = *this;
    copy.status = ManifestStatus::kDeleted;
    return copy;
  }
 
  bool operator==(const ManifestEntry& other) const;
 
  static std::shared_ptr<StructType> TypeFromPartitionType(
      std::shared_ptr<StructType> partition_type);
  static std::shared_ptr<StructType> TypeFromDataFileType(
      std::shared_ptr<StructType> datafile_type);
};
 
ICEBERG_EXPORT constexpr std::string_view ToString(DataFile::Content type) noexcept {
  switch (type) {
    case DataFile::Content::kData:
      return "data";
    case DataFile::Content::kPositionDeletes:
      return "position_deletes";
    case DataFile::Content::kEqualityDeletes:
      return "equality_deletes";
  }
  std::unreachable();
}
 
ICEBERG_EXPORT constexpr Result<DataFile::Content> DataFileContentFromInt(
    int32_t content) noexcept {
  switch (content) {
    case 0:
      return DataFile::Content::kData;
    case 1:
      return DataFile::Content::kPositionDeletes;
    case 2:
      return DataFile::Content::kEqualityDeletes;
    default:
      return InvalidArgument("Invalid data file content: {}", content);
  }
}
 
}  // namespace iceberg