Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate Class Reference

Inheritance diagram for Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate:

Classes
struct	TPrimitiveToken

Public Member Functions
	RnnTokenizerPrivate ()
virtual	~RnnTokenizerPrivate ()
void	init (GroupConfigurationStructure &unitConfiguration)
void	tokenize (const QString &text, std::vector< std::vector< TPrimitiveToken > > &sentences)
Public Member Functions inherited from Lima::LinguisticProcessing::DeepTokenizerBase
	DeepTokenizerBase ()
virtual	~DeepTokenizerBase ()
	DeepTokenizerBase (const DeepTokenizerBase &)
DeepTokenizerBase &	operator= (const DeepTokenizerBase &)
virtual void	computeDefaultStatus (LinguisticAnalysisStructure::TStatus &curSettings)
virtual void	computeDefaultStatus (LinguisticAnalysisStructure::Token &token)

Protected Member Functions
void	append_new_word (std::vector< TPrimitiveToken > &current_sentence, const QString &current_token, int current_token_offset) const
Protected Member Functions inherited from Lima::LinguisticProcessing::ConfigurationHelper
	ConfigurationHelper (const std::string &processUnitName, const std::string &loggingCategory)
void	getStringParameter (Common::XMLConfigurationFiles::GroupConfigurationStructure &unitConfiguration, const std::string &name, std::string &value, int flags=Flags::REQUIRED, std::string default_value="")
std::string	getStringParameter (Common::XMLConfigurationFiles::GroupConfigurationStructure &unitConfiguration, const std::string &name, int flags=Flags::REQUIRED, std::string default_value="")
void	getIntParameter (Common::XMLConfigurationFiles::GroupConfigurationStructure &unitConfiguration, const std::string &name, int &value, int flags=Flags::REQUIRED, int default_value=0)
int	getIntParameter (Common::XMLConfigurationFiles::GroupConfigurationStructure &unitConfiguration, const std::string &name, int flags=Flags::REQUIRED, int default_value=0)
bool	isInitLazy ()

Protected Attributes
MediaId	m_language
FsaStringsPool *	m_stringsPool
LinguisticGraphVertex	m_currentVx
QString	m_data
size_t	m_max_seq_len
std::map< QString, std::vector< QString > >	m_trrules
bool	m_ignoreEOL
std::shared_ptr< segmentation::impl::SegmentationImpl >	m_segm
std::function< void()>	m_load_fn
bool	m_loaded

Additional Inherited Members
Protected Types inherited from Lima::LinguisticProcessing::ConfigurationHelper
enum	Flags { REQUIRED = 0x1 , NOT_EMPTY = 0x2 , NOT_ZERO = 0x2 }

Detailed Description

Definition at line 62 of file RnnTokenizer.cpp.

Constructor & Destructor Documentation

RnnTokenizerPrivate()

Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::RnnTokenizerPrivate

(

)

Definition at line 109 of file RnnTokenizer.cpp.

~RnnTokenizerPrivate()

Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::~RnnTokenizerPrivate ( )

virtual

Definition at line 119 of file RnnTokenizer.cpp.

Member Function Documentation

append_new_word()

void Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::append_new_word ( std::vector< TPrimitiveToken > &  current_sentence, const QString &  current_token, int  current_token_offset  ) const

protected

Definition at line 289 of file RnnTokenizer.cpp.

init()

void Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::init

(

GroupConfigurationStructure &

unitConfiguration

)

Definition at line 237 of file RnnTokenizer.cpp.

tokenize()

void Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::tokenize	(	const QString &	text,
		std::vector< std::vector< TPrimitiveToken > > &	sentences
	)

Definition at line 317 of file RnnTokenizer.cpp.

Member Data Documentation

m_currentVx

LinguisticGraphVertex Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_currentVx

protected

Definition at line 93 of file RnnTokenizer.cpp.

m_data

QString Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_data

protected

Definition at line 94 of file RnnTokenizer.cpp.

m_ignoreEOL

bool Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_ignoreEOL

protected

Definition at line 101 of file RnnTokenizer.cpp.

m_language

MediaId Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_language

protected

Definition at line 91 of file RnnTokenizer.cpp.

m_load_fn

std::function<void()> Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_load_fn

protected

Definition at line 105 of file RnnTokenizer.cpp.

m_loaded

bool Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_loaded

protected

Definition at line 106 of file RnnTokenizer.cpp.

m_max_seq_len

size_t Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_max_seq_len

protected

Definition at line 96 of file RnnTokenizer.cpp.

m_segm

std::shared_ptr<segmentation::impl::SegmentationImpl> Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_segm

protected

Definition at line 103 of file RnnTokenizer.cpp.

m_stringsPool

FsaStringsPool* Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_stringsPool

protected

Definition at line 92 of file RnnTokenizer.cpp.

m_trrules

std::map<QString, std::vector<QString> > Lima::LinguisticProcessing::DeepLimaUnits::RnnTokenizer::RnnTokenizerPrivate::m_trrules

protected

Definition at line 98 of file RnnTokenizer.cpp.

---

The documentation for this class was generated from the following file:

• lima_linguisticprocessing/src/linguisticProcessing/core/DeepLimaUnits/RnnTokenizer/RnnTokenizer.cpp