byte_pair_encoding.hpp File Reference

#include <cudf/column/column.hpp>
#include <cudf/column/column_view.hpp>
#include <cudf/scalar/scalar.hpp>
#include <cudf/strings/strings_column_view.hpp>
#include <cudf/utilities/default_stream.hpp>

Go to the source code of this file.

Classes
struct	nvtext::bpe_merge_pairs
	The table of merge pairs for the BPE encoder. More...

Namespaces
	nvtext
	NVText APIs.

Functions
std::unique_ptr< bpe_merge_pairs >	nvtext::load_merge_pairs_file (std::string const &filename_merges, rmm::mr::device_memory_resource *mr=rmm::mr::get_current_device_resource())
	Create a nvtext::bpe_merge_pairs from an input file. More...
std::unique_ptr< bpe_merge_pairs >	nvtext::load_merge_pairs (cudf::strings_column_view const &merge_pairs, rmm::cuda_stream_view stream=cudf::get_default_stream(), rmm::mr::device_memory_resource *mr=rmm::mr::get_current_device_resource())
	Create a nvtext::bpe_merge_pairs from a strings column. More...
std::unique_ptr< cudf::column >	nvtext::byte_pair_encoding (cudf::strings_column_view const &input, bpe_merge_pairs const &merges_pairs, cudf::string_scalar const &separator=cudf::string_scalar(" "), rmm::mr::device_memory_resource *mr=rmm::mr::get_current_device_resource())
	Byte pair encode the input strings. More...