manifest_group.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <functional>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
 
#include "iceberg/delete_file_index.h"
#include "iceberg/iceberg_export.h"
#include "iceberg/manifest/manifest_entry.h"
#include "iceberg/manifest/manifest_list.h"
#include "iceberg/result.h"
#include "iceberg/table_scan.h"
#include "iceberg/type_fwd.h"
#include "iceberg/util/error_collector.h"
#include "iceberg/util/executor.h"
 
namespace iceberg {
 
struct ICEBERG_EXPORT TaskContext {
 public:
  std::shared_ptr<PartitionSpec> spec;
  DeleteFileIndex* deletes;
  ResidualEvaluator* residuals;
  bool drop_stats;
  std::unordered_set<int32_t> columns_to_keep_stats;
};
 
class ICEBERG_EXPORT ManifestGroup : public ErrorCollector {
 public:
  static Result<std::unique_ptr<ManifestGroup>> Make(
      std::shared_ptr<FileIO> io, std::shared_ptr<Schema> schema,
      std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id_,
      std::vector<ManifestFile> manifests);
 
  static Result<std::unique_ptr<ManifestGroup>> Make(
      std::shared_ptr<FileIO> io, std::shared_ptr<Schema> schema,
      std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id,
      std::vector<ManifestFile> data_manifests,
      std::vector<ManifestFile> delete_manifests);
 
  ~ManifestGroup() override;
 
  ManifestGroup(ManifestGroup&&) noexcept;
  ManifestGroup& operator=(ManifestGroup&&) noexcept;
  ManifestGroup(const ManifestGroup&) = delete;
  ManifestGroup& operator=(const ManifestGroup&) = delete;
 
  ManifestGroup& FilterData(std::shared_ptr<Expression> filter);
 
  ManifestGroup& FilterFiles(std::shared_ptr<Expression> filter);
 
  ManifestGroup& FilterPartitions(std::shared_ptr<Expression> filter);
 
  ManifestGroup& FilterManifestEntries(
      std::function<bool(const ManifestEntry&)> predicate);
 
  ManifestGroup& IgnoreDeleted();
 
  ManifestGroup& IgnoreExisting();
 
  ManifestGroup& IgnoreResiduals();
 
  ManifestGroup& Select(std::vector<std::string> columns);
 
  ManifestGroup& CaseSensitive(bool case_sensitive);
 
  ManifestGroup& ColumnsToKeepStats(std::unordered_set<int32_t> column_ids);
 
  ManifestGroup& PlanWith(OptionalExecutor executor);
 
  ManifestGroup& WithScanMetrics(std::shared_ptr<ScanMetrics> scan_metrics);
 
  Result<std::vector<std::shared_ptr<FileScanTask>>> PlanFiles() &&;
 
  Result<FileScanTaskStreamPtr> PlanFilesStream() &&;
 
  Result<std::vector<ManifestEntry>> Entries();
 
  using CreateTasksFunction =
      std::function<Result<std::vector<std::shared_ptr<ScanTask>>>(
          std::vector<ManifestEntry>&&, const TaskContext&)>;
 
  Result<std::vector<std::shared_ptr<ScanTask>>> Plan(
      const CreateTasksFunction& create_tasks);
 
 private:
  class PlanningContext;
  class FilePlanningStream;
 
  struct StatsProjection {
    std::vector<std::string> columns;
    bool drop_stats;
  };
 
  ManifestGroup(std::shared_ptr<FileIO> io, std::shared_ptr<Schema> schema,
                std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id,
                std::vector<ManifestFile> data_manifests,
                DeleteFileIndex::Builder&& delete_index_builder);
 
  Result<std::unordered_map<int32_t, std::vector<ManifestEntry>>> ReadEntries(
      PlanningContext& context);
 
  Result<std::unique_ptr<ManifestReader>> MakeReader(
      const ManifestFile& manifest, const std::vector<std::string>& columns);
 
  StatsProjection PrepareStatsProjection(bool has_equality_deletes) const;
 
  std::shared_ptr<FileIO> io_;
  std::shared_ptr<Schema> schema_;
  std::unordered_map<int32_t, std::shared_ptr<PartitionSpec>> specs_by_id_;
  std::vector<ManifestFile> data_manifests_;
  DeleteFileIndex::Builder delete_index_builder_;
  std::shared_ptr<Expression> data_filter_;
  std::shared_ptr<Expression> file_filter_;
  std::shared_ptr<Expression> partition_filter_;
  std::function<bool(const ManifestEntry&)> manifest_entry_predicate_;
  std::vector<std::string> columns_;
  std::unordered_set<int32_t> columns_to_keep_stats_;
  OptionalExecutor executor_;
  bool case_sensitive_ = true;
  bool ignore_deleted_ = false;
  bool ignore_existing_ = false;
  bool ignore_residuals_ = false;
  std::shared_ptr<ScanMetrics> scan_metrics_;
};
 
}  // namespace iceberg