file_reader.h

Go to the documentation of this file.

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
 
#pragma once
 
 
#include <cstdint>
#include <functional>
#include <memory>
#include <optional>
 
#include "iceberg/arrow_c_data.h"
#include "iceberg/file_format.h"
#include "iceberg/result.h"
#include "iceberg/type_fwd.h"
#include "iceberg/util/config.h"
 
namespace iceberg {
 
class ICEBERG_EXPORT Reader {
 public:
  virtual ~Reader() = default;
  Reader() = default;
  Reader(const Reader&) = delete;
  Reader& operator=(const Reader&) = delete;
 
  virtual Status Open(const ReaderOptions& options) = 0;
 
  virtual Status Close() = 0;
 
  virtual Result<std::optional<ArrowArray>> Next() = 0;
 
  virtual Result<ArrowSchema> Schema() = 0;
 
  virtual Result<std::unordered_map<std::string, std::string>> Metadata() = 0;
};
 
struct ICEBERG_EXPORT Split {
  size_t offset;
  size_t length;
};
 
class ICEBERG_EXPORT ReaderProperties : public ConfigBase<ReaderProperties> {
 public:
  template <typename T>
  using Entry = const ConfigBase<ReaderProperties>::Entry<T>;
 
  inline static Entry<int64_t> kBatchSize{"read.batch-size", 4096};
  inline static Entry<bool> kArrowUseLargeList{"read.arrow.use-large-list", false};
  inline static Entry<bool> kArrowValidateBatch{"read.arrow.validate-batch", true};
  inline static Entry<bool> kParquetRowGroupFilter{
      "read.parquet.row-group-filter.enabled", true};
  inline static Entry<bool> kAvroSkipDatum{"read.avro.skip-datum", true};
  inline static Entry<int64_t> kAvroBufferSize{"read.avro.buffer-size", 1024 * 1024};
 
  static ReaderProperties FromMap(
      const std::unordered_map<std::string, std::string>& properties);
};
 
struct ICEBERG_EXPORT ReaderOptions {
  std::string path;
  std::optional<size_t> length;
  std::optional<Split> split;
  std::shared_ptr<class FileIO> io;
  std::shared_ptr<class Schema> projection;
  std::shared_ptr<class Expression> filter;
  bool case_sensitive = true;
  std::shared_ptr<class NameMapping> name_mapping;
  std::optional<int64_t> first_row_id;
  std::optional<int64_t> data_sequence_number;
  ReaderProperties properties;
};
 
using ReaderFactory = std::function<Result<std::unique_ptr<Reader>>()>;
 
struct ICEBERG_EXPORT ReaderFactoryRegistry {
  ReaderFactoryRegistry(FileFormatType format_type, ReaderFactory factory);
 
  static ReaderFactory& GetFactory(FileFormatType format_type);
 
  static Result<std::unique_ptr<Reader>> Open(FileFormatType format_type,
                                              const ReaderOptions& options);
};
 
}  // namespace iceberg