rewrite_manifests.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <functional>
#include <memory>
#include <span>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
#include "iceberg/iceberg_export.h"
#include "iceberg/result.h"
#include "iceberg/type_fwd.h"
#include "iceberg/update/snapshot_update.h"
 
namespace iceberg {
 
class ICEBERG_EXPORT RewriteManifests : public SnapshotUpdate {
 public:
  using ClusterByFunc = std::function<std::string(const DataFile&)>;
  using RewritePredicate = std::function<bool(const ManifestFile&)>;
 
  static Result<std::shared_ptr<RewriteManifests>> Make(
      std::string table_name, std::shared_ptr<TransactionContext> ctx);
 
  ~RewriteManifests() override;
 
  RewriteManifests& ClusterBy(ClusterByFunc func);
 
  RewriteManifests& RewriteIf(RewritePredicate predicate);
 
  RewriteManifests& DeleteManifest(const ManifestFile& manifest);
 
  RewriteManifests& AddManifest(const ManifestFile& manifest);
 
 protected:
  std::string operation() override;
 
  Result<std::vector<ManifestFile>> Apply(
      const TableMetadata& metadata_to_update,
      const std::shared_ptr<Snapshot>& snapshot) override;
  std::unordered_map<std::string, std::string> Summary() override;
  Status CleanUncommitted(const std::unordered_set<std::string>& committed) override;
 
  explicit RewriteManifests(std::string table_name,
                            std::shared_ptr<TransactionContext> ctx);
 
 private:
  struct WriterKey {
    std::string cluster_key;
    int32_t partition_spec_id;
 
    friend bool operator==(const WriterKey&, const WriterKey&) = default;
  };
 
  struct WriterKeyHash {
    size_t operator()(const WriterKey& key) const;
  };
 
  class WriterWrapper;
 
  bool RequiresRewrite(const std::unordered_set<ManifestFile>& current_manifests) const;
  bool MatchesPredicate(const ManifestFile& manifest) const;
  bool ContainsDeletes(const ManifestFile& manifest) const;
  void KeepActiveManifests(std::span<const ManifestFile> current_manifests);
  Status ValidateDeletedManifests(
      const std::unordered_set<ManifestFile>& current_manifests,
      int64_t current_snapshot_id) const;
  Status ValidateFilesCounts() const;
 
  Result<ManifestFile> CopyManifest(const ManifestFile& manifest);
  Status PerformRewrite(std::span<const ManifestFile> current_manifests);
  Status AppendEntry(const ManifestEntry& entry, const ManifestFile& manifest,
                     const std::shared_ptr<PartitionSpec>& spec);
  Result<WriterWrapper*> GetWriter(const WriterKey& key,
                                   const std::shared_ptr<PartitionSpec>& spec);
  Result<std::unique_ptr<ManifestWriter>> NewManifestWriter(
      const std::shared_ptr<PartitionSpec>& spec);
  Status CloseWriters();
 
  void DeleteUncommitted(std::vector<ManifestFile>& manifests,
                         const std::unordered_set<std::string>& committed);
  void Reset();
 
 private:
  std::unordered_set<ManifestFile> deleted_manifests_;
  std::vector<ManifestFile> added_manifests_;
  std::vector<ManifestFile> added_manifests_to_copy_;
  std::vector<ManifestFile> rewritten_added_manifests_;
 
  std::vector<ManifestFile> kept_manifests_;
  std::vector<ManifestFile> new_manifests_;
  std::unordered_set<ManifestFile> rewritten_manifests_;
  std::unordered_map<WriterKey, std::unique_ptr<WriterWrapper>, WriterKeyHash> writers_;
 
  int64_t entry_count_{0};
 
  ClusterByFunc cluster_by_func_;
  RewritePredicate predicate_;
  bool rewrite_dirty_{true};
};
 
}  // namespace iceberg