#include <deeplima/token_sequence_analyzer.h>

Inheritance diagram for deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >:

Classes
class	enriched_token_buffer_t

class	enriched_token_t

struct	lemm_cache_key_hash

class	TokenIterator

Public Types
typedef TokenIterator	output_iterator_t

Public Types inherited from deeplima::ITokenSequenceAnalyzer
typedef std::function< void(std::shared_ptr< StringIndex > stridx, const token_buffer_t<> &tokens, const std::vector< StringIndex::idx_t > &lemmata, std::shared_ptr< StdMatrix< uint8_t > > classes, size_t begin, size_t end) >	output_callback_t

Public Member Functions
	TokenSequenceAnalyzer ()

	TokenSequenceAnalyzer (const std::string &model_fn, const std::string &lemm_model_fn, const std::string &lemm_dict_fn, const std::string &fixed_ini_fn, const std::string &lower_ini_fn, const std::string &fixed_lemm_fn, const PathResolver &path_resolver, size_t buffer_size, size_t num_buffers)

void	get_classes_from_fn (const std::string &fn, std::vector< std::string > &classes_names, std::vector< std::vector< std::string > > &classes)

virtual	~TokenSequenceAnalyzer ()

virtual void	register_handler (const output_callback_t fn) override

virtual const std::vector< std::vector< std::string > > &	get_classes () const override

virtual const std::vector< std::string > &	get_class_names () const override

virtual std::shared_ptr< StringIndex >	get_stridx () const override

virtual void	finalize () override

virtual void	operator() (const std::vector< deeplima::segmentation::token_pos > &tokens, uint32_t len) override

Protected Types
typedef std::pair< StringIndex::idx_t, morph_model::morph_feats_t >	lemm_cache_key_t

Protected Member Functions
void	acquire_buffer ()

void	start_analysis (size_t buffer_idx, int count=-1)

void	process_buffer (size_t buffer_idx)

void	load_lemm_cache (const std::string &fn)

std::unordered_set< morph_model::morph_feats_t >	load_pos_cache (const std::string &fn)

void	lemmatize (const token_buffer_t<> &buffer, std::vector< StringIndex::idx_t > &lemm_buffer, std::shared_ptr< StdMatrix< uint8_t > > classes, size_t offset, size_t end)
	TODO correct this function.

Static Protected Member Functions
static std::u32string	to_lower (const std::u32string &src)
	This well lower onlu Latin1 characters.

Protected Attributes
size_t	m_buffer_size

size_t	m_current_buffer

size_t	m_current_timepoint

std::shared_ptr< StringIndex >	m_stridx_ptr

StringIndex &	m_stridx

StringIndex::idx_t	m_unk_idx

std::vector< token_buffer_t<> >	m_buffers

std::vector< std::vector< StringIndex::idx_t > >	m_lemm_buffers

tagging::impl::TaggingImpl< TaggingAuxScalar >	m_cls

lemmatization::impl::LemmatizationImpl	m_lemm

std::shared_ptr< StdMatrix< uint8_t > >	m_classes

std::unordered_map< lemm_cache_key_t, StringIndex::idx_t, lemm_cache_key_hash >	m_lemm_cache

std::unordered_set< morph_model::morph_feats_t >	m_fixed_ini_cache

std::unordered_set< morph_model::morph_feats_t >	m_lower_ini_cache

std::unordered_set< morph_model::morph_feats_t >	m_fixed_lemm_cache

output_callback_t	m_output_callback

Detailed Description

template<typename TaggingAuxScalar = float>
class deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >

Definition at line 77 of file token_sequence_analyzer.h.

Member Typedef Documentation

◆ lemm_cache_key_t

template<typename TaggingAuxScalar = float>

typedef std::pair<StringIndex::idx_t, morph_model::morph_feats_t> deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::lemm_cache_key_t

protected

Definition at line 552 of file token_sequence_analyzer.h.

◆ output_iterator_t

template<typename TaggingAuxScalar = float>

typedef TokenIterator deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::output_iterator_t

Definition at line 202 of file token_sequence_analyzer.h.

Constructor & Destructor Documentation

◆ TokenSequenceAnalyzer() [1/2]

template<typename TaggingAuxScalar = float>

deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::TokenSequenceAnalyzer ( )

inline

Definition at line 293 of file token_sequence_analyzer.h.

◆ TokenSequenceAnalyzer() [2/2]

template<typename TaggingAuxScalar = float>

deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::TokenSequenceAnalyzer	(	const std::string &	model_fn,
		const std::string &	lemm_model_fn,
		const std::string &	lemm_dict_fn,
		const std::string &	fixed_ini_fn,
		const std::string &	lower_ini_fn,
		const std::string &	fixed_lemm_fn,
		const PathResolver &	path_resolver,
		size_t	buffer_size,
		size_t	num_buffers
	)

inline

Definition at line 303 of file token_sequence_analyzer.h.

◆ ~TokenSequenceAnalyzer()

template<typename TaggingAuxScalar = float>

virtual deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::~TokenSequenceAnalyzer ( )

inlinevirtual

Definition at line 424 of file token_sequence_analyzer.h.

Member Function Documentation

◆ acquire_buffer()

template<typename TaggingAuxScalar = float>

void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::acquire_buffer ( )

inlineprotected

Definition at line 519 of file token_sequence_analyzer.h.

◆ finalize()

template<typename TaggingAuxScalar = float>

virtual void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::finalize ( )

inlineoverridevirtual

Implements deeplima::ITokenSequenceAnalyzer.

Definition at line 456 of file token_sequence_analyzer.h.

◆ get_class_names()

template<typename TaggingAuxScalar = float>

virtual const std::vector< std::string > & deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::get_class_names ( ) const

inlineoverridevirtual

Implements deeplima::ITokenSequenceAnalyzer.

Definition at line 439 of file token_sequence_analyzer.h.

◆ get_classes()

template<typename TaggingAuxScalar = float>

virtual const std::vector< std::vector< std::string > > & deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::get_classes ( ) const

inlineoverridevirtual

Implements deeplima::ITokenSequenceAnalyzer.

Definition at line 434 of file token_sequence_analyzer.h.

◆ get_classes_from_fn()

template<typename TaggingAuxScalar = float>

void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::get_classes_from_fn	(	const std::string &	fn,
		std::vector< std::string > &	classes_names,
		std::vector< std::vector< std::string > > &	classes
	)

inline

Definition at line 417 of file token_sequence_analyzer.h.

◆ get_stridx()

template<typename TaggingAuxScalar = float>

virtual std::shared_ptr< StringIndex > deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::get_stridx ( ) const

inlineoverridevirtual

Implements deeplima::ITokenSequenceAnalyzer.

Definition at line 444 of file token_sequence_analyzer.h.

◆ lemmatize()

template<typename TaggingAuxScalar = float>

void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::lemmatize	(	const token_buffer_t<> &	buffer,
		std::vector< StringIndex::idx_t > &	lemm_buffer,
		std::shared_ptr< StdMatrix< uint8_t > >	classes,
		size_t	offset,
		size_t	end
	)

inlineprotected

TODO correct this function.

It currently returns only empty strings TODO use this function instead of the one above as soon as it is corrected

Definition at line 677 of file token_sequence_analyzer.h.

◆ load_lemm_cache()

template<typename TaggingAuxScalar = float>

void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::load_lemm_cache ( const std::string & fn )

inlineprotected

Definition at line 560 of file token_sequence_analyzer.h.

◆ load_pos_cache()

template<typename TaggingAuxScalar = float>

std::unordered_set< morph_model::morph_feats_t > deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::load_pos_cache ( const std::string & fn )

inlineprotected

Definition at line 617 of file token_sequence_analyzer.h.

◆ operator()()

template<typename TaggingAuxScalar = float>

virtual void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::operator()	(	const std::vector< deeplima::segmentation::token_pos > &	tokens,
		uint32_t	len
	)

inlineoverridevirtual

Implements deeplima::ITokenSequenceAnalyzer.

Definition at line 478 of file token_sequence_analyzer.h.

◆ process_buffer()

template<typename TaggingAuxScalar = float>

void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::process_buffer ( size_t buffer_idx )

inlineprotected

Definition at line 546 of file token_sequence_analyzer.h.

◆ register_handler()

template<typename TaggingAuxScalar = float>

virtual void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::register_handler ( const output_callback_t fn )

inlineoverridevirtual

Implements deeplima::ITokenSequenceAnalyzer.

Definition at line 428 of file token_sequence_analyzer.h.

◆ start_analysis()

template<typename TaggingAuxScalar = float>

void deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::start_analysis	(	size_t	buffer_idx,
		int	count = `-1`
	)

inlineprotected

Definition at line 536 of file token_sequence_analyzer.h.

◆ to_lower()

template<typename TaggingAuxScalar = float>

static std::u32string deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::to_lower ( const std::u32string & src )

inlinestaticprotected

This well lower onlu Latin1 characters.

Definition at line 646 of file token_sequence_analyzer.h.

Member Data Documentation

◆ m_buffer_size

template<typename TaggingAuxScalar = float>

size_t deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_buffer_size

protected

Definition at line 746 of file token_sequence_analyzer.h.

◆ m_buffers

template<typename TaggingAuxScalar = float>

std::vector<token_buffer_t<> > deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_buffers

protected

Definition at line 753 of file token_sequence_analyzer.h.

◆ m_classes

template<typename TaggingAuxScalar = float>

std::shared_ptr< StdMatrix<uint8_t> > deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_classes

protected

Definition at line 758 of file token_sequence_analyzer.h.

◆ m_cls

template<typename TaggingAuxScalar = float>

tagging::impl::TaggingImpl<TaggingAuxScalar> deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_cls

protected

Definition at line 756 of file token_sequence_analyzer.h.

◆ m_current_buffer

template<typename TaggingAuxScalar = float>

size_t deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_current_buffer

protected

Definition at line 747 of file token_sequence_analyzer.h.

◆ m_current_timepoint

template<typename TaggingAuxScalar = float>

size_t deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_current_timepoint

protected

Definition at line 748 of file token_sequence_analyzer.h.

◆ m_fixed_ini_cache

template<typename TaggingAuxScalar = float>

std::unordered_set<morph_model::morph_feats_t> deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_fixed_ini_cache

protected

Definition at line 765 of file token_sequence_analyzer.h.

◆ m_fixed_lemm_cache

template<typename TaggingAuxScalar = float>

std::unordered_set<morph_model::morph_feats_t> deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_fixed_lemm_cache

protected

Definition at line 767 of file token_sequence_analyzer.h.

◆ m_lemm

template<typename TaggingAuxScalar = float>

lemmatization::impl::LemmatizationImpl deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_lemm

protected

Definition at line 757 of file token_sequence_analyzer.h.

◆ m_lemm_buffers

template<typename TaggingAuxScalar = float>

std::vector<std::vector<StringIndex::idx_t> > deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_lemm_buffers

protected

Definition at line 754 of file token_sequence_analyzer.h.

◆ m_lemm_cache

template<typename TaggingAuxScalar = float>

std::unordered_map<lemm_cache_key_t, StringIndex::idx_t, lemm_cache_key_hash> deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_lemm_cache

protected

Definition at line 760 of file token_sequence_analyzer.h.

◆ m_lower_ini_cache

template<typename TaggingAuxScalar = float>

std::unordered_set<morph_model::morph_feats_t> deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_lower_ini_cache

protected

Definition at line 766 of file token_sequence_analyzer.h.

◆ m_output_callback

template<typename TaggingAuxScalar = float>

output_callback_t deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_output_callback

protected

Definition at line 769 of file token_sequence_analyzer.h.

◆ m_stridx

template<typename TaggingAuxScalar = float>

StringIndex& deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_stridx

protected

Definition at line 751 of file token_sequence_analyzer.h.

◆ m_stridx_ptr

template<typename TaggingAuxScalar = float>

std::shared_ptr<StringIndex> deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_stridx_ptr

protected

Definition at line 750 of file token_sequence_analyzer.h.

◆ m_unk_idx

template<typename TaggingAuxScalar = float>

StringIndex::idx_t deeplima::TokenSequenceAnalyzer< TaggingAuxScalar >::m_unk_idx

protected

Definition at line 752 of file token_sequence_analyzer.h.

The documentation for this class was generated from the following file:

deeplima/include/deeplima/token_sequence_analyzer.h

Classes

Public Types

Public Member Functions

Protected Types

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Detailed Description

Member Typedef Documentation

◆ lemm_cache_key_t

◆ output_iterator_t

Constructor & Destructor Documentation

◆ TokenSequenceAnalyzer() [1/2]

◆ TokenSequenceAnalyzer() [2/2]

◆ ~TokenSequenceAnalyzer()

Member Function Documentation

◆ acquire_buffer()

◆ finalize()

◆ get_class_names()

◆ get_classes()

◆ get_classes_from_fn()

◆ get_stridx()

◆ lemmatize()

◆ load_lemm_cache()

◆ load_pos_cache()

◆ operator()()

◆ process_buffer()

◆ register_handler()

◆ start_analysis()

◆ to_lower()

Member Data Documentation

◆ m_buffer_size

◆ m_buffers

◆ m_classes

◆ m_cls

◆ m_current_buffer

◆ m_current_timepoint

◆ m_fixed_ini_cache

◆ m_fixed_lemm_cache

◆ m_lemm

◆ m_lemm_buffers

◆ m_lemm_cache

◆ m_lower_ini_cache

◆ m_output_callback

◆ m_stridx

◆ m_stridx_ptr

◆ m_unk_idx