orc_metadata.hpp

Go to the documentation of this file.

/*
 * Copyright (c) 2019-2023, NVIDIA CORPORATION.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#pragma once
 
#include <cudf/io/orc_types.hpp>
#include <cudf/io/types.hpp>
 
#include <optional>
#include <variant>
#include <vector>
 
namespace cudf {
namespace io {
 
struct raw_orc_statistics {
  std::vector<std::string> column_names;                
  std::vector<std::string> file_stats;                  
  std::vector<std::vector<std::string>> stripes_stats;  
};
 
raw_orc_statistics read_raw_orc_statistics(source_info const& src_info);
 
using no_statistics = std::monostate;
 
template <typename T>
struct minmax_statistics {
  std::optional<T> minimum;  
  std::optional<T> maximum;  
};
 
template <typename T>
struct sum_statistics {
  std::optional<T> sum;  
};
 
struct integer_statistics : minmax_statistics<int64_t>, sum_statistics<int64_t> {};
 
struct double_statistics : minmax_statistics<double>, sum_statistics<double> {};
 
struct string_statistics : minmax_statistics<std::string>, sum_statistics<int64_t> {};
 
struct bucket_statistics {
  std::vector<uint64_t> count;  
};
 
struct decimal_statistics : minmax_statistics<std::string>, sum_statistics<std::string> {};
 
using date_statistics = minmax_statistics<int32_t>;
 
using binary_statistics = sum_statistics<int64_t>;
 
struct timestamp_statistics : minmax_statistics<int64_t> {
  std::optional<int64_t> minimum_utc;     
  std::optional<int64_t> maximum_utc;     
  std::optional<uint32_t> minimum_nanos;  
  std::optional<uint32_t> maximum_nanos;  
};
 
namespace orc {
// forward declare the type that ProtobufReader uses. The `cudf::io::column_statistics` objects,
// returned from `read_parsed_orc_statistics`, are constructed from
// `cudf::io::orc::column_statistics` objects that `ProtobufReader` initializes.
struct column_statistics;
}  // namespace orc
 
struct column_statistics {
  std::optional<uint64_t> number_of_values;  
  std::optional<bool> has_null;              
  std::variant<no_statistics,
               integer_statistics,
               double_statistics,
               string_statistics,
               bucket_statistics,
               decimal_statistics,
               date_statistics,
               binary_statistics,
               timestamp_statistics>
    type_specific_stats;  
 
  column_statistics(orc::column_statistics&& detail_statistics);
};
 
struct parsed_orc_statistics {
  std::vector<std::string> column_names;                      
  std::vector<column_statistics> file_stats;                  
  std::vector<std::vector<column_statistics>> stripes_stats;  
};
 
parsed_orc_statistics read_parsed_orc_statistics(source_info const& src_info);
 
struct orc_column_schema {
 public:
  orc_column_schema(std::string_view name,
                    orc::TypeKind type,
                    std::vector<orc_column_schema> children)
    : _name{name}, _type_kind{type}, _children{std::move(children)}
  {
  }
 
  [[nodiscard]] auto name() const { return _name; }
 
  [[nodiscard]] auto type_kind() const { return _type_kind; }
 
  [[nodiscard]] auto const& children() const& { return _children; }
 
  [[nodiscard]] auto children() && { return std::move(_children); }
 
  [[nodiscard]] auto const& child(int idx) const& { return children().at(idx); }
 
  [[nodiscard]] auto child(int idx) && { return std::move(children().at(idx)); }
 
  [[nodiscard]] auto num_children() const { return children().size(); }
 
 private:
  std::string _name;
  orc::TypeKind _type_kind;
  std::vector<orc_column_schema> _children;
};
 
struct orc_schema {
 public:
  orc_schema(orc_column_schema root_column_schema) : _root{std::move(root_column_schema)} {}
 
  [[nodiscard]] auto const& root() const& { return _root; }
 
  [[nodiscard]] auto root() && { return std::move(_root); }
 
 private:
  orc_column_schema _root;
};
 
class orc_metadata {
 public:
  orc_metadata(orc_schema schema, size_type num_rows, size_type num_stripes)
    : _schema{std::move(schema)}, _num_rows{num_rows}, _num_stripes{num_stripes}
  {
  }
 
  [[nodiscard]] auto const& schema() const { return _schema; }
 
 
  [[nodiscard]] auto num_rows() const { return _num_rows; }
 
  [[nodiscard]] auto num_stripes() const { return _num_stripes; }
 
 private:
  orc_schema _schema;
  size_type _num_rows;
  size_type _num_stripes;
};
 
orc_metadata read_orc_metadata(source_info const& src_info);
 
}  // namespace io
}  // namespace cudf