delete_filter.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <atomic>
#include <cstdint>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <span>
#include <string>
#include <vector>
 
#include "iceberg/arrow_c_data.h"
#include "iceberg/data/delete_loader.h"
#include "iceberg/deletes/position_delete_index.h"
#include "iceberg/iceberg_data_export.h"
#include "iceberg/result.h"
#include "iceberg/schema_field.h"
#include "iceberg/type_fwd.h"
 
namespace iceberg {
 
struct ICEBERG_DATA_EXPORT AliveRowSelection {
  std::vector<int32_t> indices;
 
  int64_t alive_count() const { return static_cast<int64_t>(indices.size()); }
 
  bool empty() const { return indices.empty(); }
};
 
class ICEBERG_DATA_EXPORT DeleteCounter {
 public:
  void Increment(int64_t count = 1) {
    count_.fetch_add(count, std::memory_order_relaxed);
  }
  int64_t Get() const { return count_.load(std::memory_order_relaxed); }
 
 private:
  std::atomic<int64_t> count_{0};
};
 
class ICEBERG_DATA_EXPORT DeleteFilter {
 public:
  struct FieldLookupResult {
    SchemaField field;
    SchemaField projection_field;
  };
 
  using FieldLookup = std::function<Result<std::optional<FieldLookupResult>>(int32_t)>;
 
  static Result<FieldLookup> MakeFieldLookup(
      std::shared_ptr<Schema> table_schema,
      std::span<const std::shared_ptr<Schema>> schemas = {});
 
  static Result<FieldLookup> MakeFieldLookup(
      std::shared_ptr<TableMetadata> table_metadata);
 
  static Result<std::unique_ptr<DeleteFilter>> Make(
      std::string file_path, std::span<const std::shared_ptr<DataFile>> delete_files,
      std::shared_ptr<Schema> table_schema, std::shared_ptr<Schema> requested_schema,
      std::shared_ptr<FileIO> io, bool need_row_pos_col = true,
      std::shared_ptr<DeleteCounter> counter = nullptr);
 
  static Result<std::unique_ptr<DeleteFilter>> Make(
      std::string file_path, std::span<const std::shared_ptr<DataFile>> delete_files,
      std::shared_ptr<TableMetadata> table_metadata,
      std::shared_ptr<Schema> requested_schema, std::shared_ptr<FileIO> io,
      bool need_row_pos_col = true, std::shared_ptr<DeleteCounter> counter = nullptr);
 
  static Result<std::unique_ptr<DeleteFilter>> Make(
      std::string file_path, std::span<const std::shared_ptr<DataFile>> delete_files,
      std::shared_ptr<Schema> table_schema, std::shared_ptr<Schema> requested_schema,
      std::shared_ptr<FileIO> io, std::span<const std::shared_ptr<Schema>> schemas,
      bool need_row_pos_col = true, std::shared_ptr<DeleteCounter> counter = nullptr);
 
  static Result<std::unique_ptr<DeleteFilter>> Make(
      std::string file_path, std::span<const std::shared_ptr<DataFile>> delete_files,
      std::shared_ptr<Schema> requested_schema, std::shared_ptr<FileIO> io,
      FieldLookup field_lookup, bool need_row_pos_col = true,
      std::shared_ptr<DeleteCounter> counter = nullptr);
 
  ~DeleteFilter();
 
  const std::shared_ptr<Schema>& RequiredSchema() const;
 
  const std::shared_ptr<Schema>& ExpectedSchema() const;
 
  void IncrementDeleteCount(int64_t count = 1);
 
  Result<const PositionDeleteIndex*> DeletedRowPositions() const;
 
  Result<std::function<Result<bool>(const StructLike&)>> EqDeletedRowFilter() const;
 
  Result<std::function<Result<bool>(const StructLike&)>> FindEqualityDeleteRows() const;
 
  Result<AliveRowSelection> ComputeAliveRows(const ArrowSchema& batch_schema,
                                             const ArrowArray& batch) const;
 
  bool HasPositionDeletes() const;
  bool HasEqualityDeletes() const;
 
  DeleteFilter(const DeleteFilter&) = delete;
  DeleteFilter& operator=(const DeleteFilter&) = delete;
 
 private:
  struct EqDeleteGroup;
 
  DeleteFilter(std::string file_path, std::shared_ptr<Schema> requested_schema,
               std::shared_ptr<FileIO> io, FieldLookup field_lookup,
               bool need_row_pos_col, std::shared_ptr<DeleteCounter> counter);
 
  Status Init(std::span<const std::shared_ptr<DataFile>> delete_files);
  Result<std::shared_ptr<Schema>> ComputeRequiredSchema() const;
  Status EnsurePositionDeletesLoaded() const;
  Status EnsureEqualityDeletesLoaded() const;
 
  const std::string file_path_;
  std::vector<std::shared_ptr<DataFile>> pos_deletes_;
  std::vector<std::shared_ptr<DataFile>> eq_deletes_;
 
  std::shared_ptr<Schema> requested_schema_;
  std::shared_ptr<Schema> required_schema_;
  FieldLookup field_lookup_;
 
  const bool need_row_pos_col_;
  // Position of `_pos` in required_schema_ when existent
  std::optional<size_t> pos_field_position_;
  std::shared_ptr<DeleteCounter> counter_;
 
  // TODO(gangwu): expose a factory hook (e.g. a std::function<DeleteLoader()> or a
  // virtual newDeleteLoader()) so callers can inject a caching DeleteLoader (analogous to
  // SparkDeleteFilter.CachingDeleteLoader in Java).
  DeleteLoader delete_loader_;
 
  mutable std::mutex pos_mutex_;
  mutable bool pos_loaded_ = false;
  mutable PositionDeleteIndex pos_index_;
 
  mutable std::mutex eq_mutex_;
  mutable bool eq_loaded_ = false;
  mutable std::vector<std::unique_ptr<EqDeleteGroup>> eq_groups_;
  mutable std::function<Result<bool>(const StructLike&)> eq_deleted_row_filter_cache_;
};
 
}  // namespace iceberg