manifest_writer.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <functional>
#include <memory>
#include <string>
#include <vector>
 
#include "iceberg/file_writer.h"
#include "iceberg/iceberg_export.h"
#include "iceberg/manifest/manifest_list.h"
#include "iceberg/metrics.h"
#include "iceberg/result.h"
#include "iceberg/type_fwd.h"
 
namespace iceberg {
 
class ICEBERG_EXPORT ManifestWriter {
 public:
  ~ManifestWriter();
 
  Status WriteEntry(const ManifestEntry& entry);
 
  Status WriteAddedEntry(std::shared_ptr<DataFile> file,
                         std::optional<int64_t> data_sequence_number = std::nullopt);
  Status WriteAddedEntry(const ManifestEntry& entry);
 
  Status WriteExistingEntry(std::shared_ptr<DataFile> file, int64_t file_snapshot_id,
                            int64_t data_sequence_number,
                            std::optional<int64_t> file_sequence_number = std::nullopt);
  Status WriteExistingEntry(const ManifestEntry& entry);
 
  Status WriteDeletedEntry(std::shared_ptr<DataFile> file, int64_t data_sequence_number,
                           std::optional<int64_t> file_sequence_number = std::nullopt);
  Status WriteDeletedEntry(const ManifestEntry& entry);
 
  Status AddAll(const std::vector<ManifestEntry>& entries);
 
  Status Close();
 
  ManifestContent content() const;
 
  Result<Metrics> metrics() const;
 
  Result<int64_t> length() const;
 
  Result<ManifestFile> ToManifestFile() const;
 
  static Result<std::unique_ptr<ManifestWriter>> MakeWriter(
      int8_t format_version, std::optional<int64_t> snapshot_id,
      std::string_view manifest_location, std::shared_ptr<FileIO> file_io,
      std::shared_ptr<PartitionSpec> partition_spec,
      std::shared_ptr<Schema> current_schema,
      ManifestContent content = ManifestContent::kData,
      std::optional<int64_t> first_row_id = std::nullopt);
 
 private:
  // Private constructor for internal use only, use the static Make*Writer methods
  // instead.
  ManifestWriter(std::unique_ptr<Writer> writer,
                 std::unique_ptr<class ManifestEntryAdapter> adapter,
                 std::string_view manifest_location, std::optional<int64_t> first_row_id);
 
  Status CheckDataFile(const DataFile& file) const;
 
  static constexpr int64_t kBatchSize = 1024;
  std::unique_ptr<Writer> writer_;
  std::unique_ptr<class ManifestEntryAdapter> adapter_;
  bool closed_{false};
  std::string manifest_location_;
  std::optional<int64_t> first_row_id_;
 
  int32_t add_files_count_{0};
  int32_t existing_files_count_{0};
  int32_t delete_files_count_{0};
  int64_t add_rows_count_{0L};
  int64_t existing_rows_count_{0L};
  int64_t delete_rows_count_{0L};
  std::optional<int64_t> min_sequence_number_{std::nullopt};
  std::unique_ptr<PartitionSummary> partition_summary_;
};
 
using ManifestWriterFactory = std::function<Result<std::unique_ptr<ManifestWriter>>(
    int32_t spec_id, ManifestContent content)>;
 
class ICEBERG_EXPORT ManifestListWriter {
 public:
  ~ManifestListWriter();
 
  Status Add(const ManifestFile& file);
 
  Status AddAll(const std::vector<ManifestFile>& files);
 
  Status Close();
 
  std::optional<int64_t> next_row_id() const;
 
  static Result<std::unique_ptr<ManifestListWriter>> MakeWriter(
      int8_t format_version, int64_t snapshot_id,
      std::optional<int64_t> parent_snapshot_id, std::string_view manifest_list_location,
      std::shared_ptr<FileIO> file_io,
      std::optional<int64_t> sequence_number = std::nullopt,
      std::optional<int64_t> first_row_id = std::nullopt);
 
 private:
  // Private constructor for internal use only, use the static Make*Writer methods
  // instead.
  ManifestListWriter(std::unique_ptr<Writer> writer,
                     std::unique_ptr<class ManifestFileAdapter> adapter);
 
  static constexpr int64_t kBatchSize = 1024;
  std::unique_ptr<Writer> writer_;
  std::unique_ptr<class ManifestFileAdapter> adapter_;
};
 
}  // namespace iceberg