The implementation of the segmenter, a SegmentationClassifier, itself a RnnSequenceClassifier. More...

#include <deeplima/segmentation/impl/segmentation_impl.h>

Inheritance diagram for deeplima::segmentation::impl::SegmentationImpl:

Public Member Functions
	SegmentationImpl ()

virtual	~SegmentationImpl ()=default

virtual void	load (const std::string &fn)

void	init (size_t threads, size_t buffer_size_per_thread)

virtual void	parse_from_stream (const read_callback_t fn) override

virtual void	register_handler (const segmentation_callback_t fn) override

virtual void	finalize () override
	Cleanup all remaining locks if any.

virtual bool	predicts_mwt () const override
	MWT-aware iff the model has more than the base segmentation tag classes (i.e.

Public Member Functions inherited from deeplima::segmentation::ISegmentation
virtual	~ISegmentation ()

Public Member Functions inherited from deeplima::RnnSequenceClassifier< Model, InputVectorizer, Out >
	RnnSequenceClassifier ()

virtual	~RnnSequenceClassifier ()

int32_t	next_slot (uint32_t idx)

std::shared_ptr< StdMatrix< Out > >	get_output ()

virtual void	reset ()
	Need to be called to be able to reuse this classifier on several sequences.

virtual void	init (uint32_t max_feat, uint32_t overlap, uint32_t num_slots, uint32_t slot_len, uint32_t num_threads, bool precomputed_input=false)

void	load (const std::string &fn)

void	get_classes_from_fn (const std::string &fn, std::vector< std::string > &classes_names, std::vector< std::vector< std::string > > &classes)

uint8_t	get_output (uint64_t pos, uint8_t cls)

uint64_t	get_slot_begin (uint32_t idx) const

bool	get_slot_started (uint32_t idx) const

uint64_t	get_slot_end (uint32_t idx) const

uint8_t	get_lock_count (uint32_t idx) const

void	increment_lock_count (uint32_t idx, uint8_t v=1)

void	decrement_lock_count (uint32_t idx)

uint64_t	get_start_timepoint () const

void	increment_timepoint (uint64_t &timepoint)

uint32_t	get_num_slots () const

uint32_t	get_slot_size () const

int32_t	get_slot_idx (uint64_t timepoint) const

void	set_slot_lengths (uint32_t idx, const std::vector< size_t > &lengths)

void	set_slot_begin (uint32_t idx, uint64_t slot_begin)

void	set_slot_end (uint32_t idx, uint64_t slot_end)

void	start_job (uint32_t idx, bool no_more_data=false)

void	wait_for_slot (uint32_t idx)

void	pretty_print () const

Public Member Functions inherited from deeplima::ThreadPool< RnnSequenceClassifier< Model, InputVectorizer, Out > >
	ThreadPool (size_t num_threads=0)

void	init (size_t num_threads)

size_t	get_num_threads () const

virtual	~ThreadPool ()

void	stop ()

size_t	running ()

void	push (void *job)

Protected Member Functions
void	vectorize_timepoint (uint64_t timepoint)

void	increment_timepoint (uint64_t &timepoint)

void	send_results (int32_t slot_idx)

void	send_next_results ()

void	acquire_slot ()

void	handle_timepoint ()

void	no_more_data ()

Protected Member Functions inherited from deeplima::RnnSequenceClassifier< Model, InputVectorizer, Out >
int32_t	prev_slot (uint32_t idx)

void	clear_slot (uint32_t idx)

void	start_job_impl (uint32_t idx)
	Push the slot idx in the thread pool for starting the job on it.

Protected Member Functions inherited from deeplima::ThreadPool< RnnSequenceClassifier< Model, InputVectorizer, Out > >
bool	wait_for_new_job (void **job)
	This will wait until a job is available and then job parameter will be set to this available which will be removed from the list.

void	wait_for_any_job_notification (const std::function< bool()> fn)

void	thread_fn (size_t worker_id)

Protected Attributes
std::vector< uint8_t >	m_char_len

InputEncoder	m_input_encoder

OutputDecoder	m_decoder

uint64_t	m_current_timepoint

uint32_t	m_current_slot_timepoints

int32_t	m_current_slot_no

int32_t	m_last_completed_slot

locked_buffer_set_t	m_buff_set

size_t	m_curr_buff_idx

Protected Attributes inherited from deeplima::RnnSequenceClassifier< Model, InputVectorizer, Out >
friend	RnnSequenceClassifierThreadPool

uint32_t	m_overlap

uint32_t	m_num_slots

uint32_t	m_slot_len

std::vector< slot_t >	m_slots

std::vector< std::vector< size_t > >	m_lengths

std::shared_ptr< StdMatrix< Out > >	m_output

Protected Attributes inherited from deeplima::ThreadPool< RnnSequenceClassifier< Model, InputVectorizer, Out > >
std::vector< std::thread >	m_workers

std::atomic< bool >	m_stop

std::queue< void * >	m_jobs

std::mutex	m_mutex

std::condition_variable	m_cv

std::mutex	m_mutex_notify

std::condition_variable	m_cv_notify

Additional Inherited Members
Public Types inherited from deeplima::segmentation::ISegmentation
typedef std::function< bool(uint8_t *buffer, int32_t &read, int32_t max) >	read_callback_t

Protected Types inherited from deeplima::RnnSequenceClassifier< Model, InputVectorizer, Out >
enum	slot_flags_t : uint8_t { none = 0x00 , left_overlap = 0x01 , right_overlap = 0x02 , max_flags }

typedef RnnSequenceClassifier< Model, InputVectorizer, Out >	ThisClass

typedef ThreadPool< ThisClass >	RnnSequenceClassifierThreadPool

Static Protected Member Functions inherited from deeplima::RnnSequenceClassifier< Model, InputVectorizer, Out >
static void	run_one_job (ThisClass this_ptr, size_t worker_id, void p)

Detailed Description

The implementation of the segmenter, a SegmentationClassifier, itself a RnnSequenceClassifier.

Definition at line 69 of file segmentation_impl.h.

Constructor & Destructor Documentation

◆ SegmentationImpl()

deeplima::segmentation::impl::SegmentationImpl::SegmentationImpl ( )

Definition at line 10 of file segmentation_impl.cpp.

◆ ~SegmentationImpl()

virtual deeplima::segmentation::impl::SegmentationImpl::~SegmentationImpl ( )

virtualdefault

Member Function Documentation

◆ acquire_slot()

void deeplima::segmentation::impl::SegmentationImpl::acquire_slot ( )

protected

Definition at line 275 of file segmentation_impl.cpp.

◆ finalize()

void deeplima::segmentation::impl::SegmentationImpl::finalize ( )

overridevirtual

Cleanup all remaining locks if any.

Implements deeplima::segmentation::ISegmentation.

Definition at line 329 of file segmentation_impl.cpp.

◆ handle_timepoint()

void deeplima::segmentation::impl::SegmentationImpl::handle_timepoint ( )

protected

Definition at line 303 of file segmentation_impl.cpp.

◆ increment_timepoint()

void deeplima::segmentation::impl::SegmentationImpl::increment_timepoint ( uint64_t & timepoint )

protected

Definition at line 184 of file segmentation_impl.cpp.

◆ init()

void deeplima::segmentation::impl::SegmentationImpl::init	(	size_t	threads,
		size_t	buffer_size_per_thread
	)

Definition at line 50 of file segmentation_impl.cpp.

◆ load()

void deeplima::segmentation::impl::SegmentationImpl::load ( const std::string & fn )

virtual

Definition at line 35 of file segmentation_impl.cpp.

◆ no_more_data()

void deeplima::segmentation::impl::SegmentationImpl::no_more_data ( )

protected

Definition at line 320 of file segmentation_impl.cpp.

◆ parse_from_stream()

void deeplima::segmentation::impl::SegmentationImpl::parse_from_stream ( const read_callback_t fn )

overridevirtual

Implements deeplima::segmentation::ISegmentation.

Definition at line 61 of file segmentation_impl.cpp.

◆ predicts_mwt()

virtual bool deeplima::segmentation::impl::SegmentationImpl::predicts_mwt ( ) const

inlineoverridevirtual

MWT-aware iff the model has more than the base segmentation tag classes (i.e.

it was trained with –mwt, giving the 4 extra _MWT end tags). The output class count is recorded as the size of the first output dict at model-conversion time.

Reimplemented from deeplima::segmentation::ISegmentation.

Definition at line 98 of file segmentation_impl.h.

◆ register_handler()

void deeplima::segmentation::impl::SegmentationImpl::register_handler ( const segmentation_callback_t fn )

overridevirtual

Implements deeplima::segmentation::ISegmentation.

Definition at line 169 of file segmentation_impl.cpp.

◆ send_next_results()

void deeplima::segmentation::impl::SegmentationImpl::send_next_results ( )

protected

Definition at line 236 of file segmentation_impl.cpp.

◆ send_results()

void deeplima::segmentation::impl::SegmentationImpl::send_results ( int32_t slot_idx )

protected

Definition at line 191 of file segmentation_impl.cpp.

◆ vectorize_timepoint()

void deeplima::segmentation::impl::SegmentationImpl::vectorize_timepoint ( uint64_t timepoint )

protected

Definition at line 174 of file segmentation_impl.cpp.

Member Data Documentation

◆ m_buff_set

locked_buffer_set_t deeplima::segmentation::impl::SegmentationImpl::m_buff_set

protected

Definition at line 132 of file segmentation_impl.h.

◆ m_char_len

std::vector<uint8_t> deeplima::segmentation::impl::SegmentationImpl::m_char_len

protected

Definition at line 121 of file segmentation_impl.h.

◆ m_curr_buff_idx

size_t deeplima::segmentation::impl::SegmentationImpl::m_curr_buff_idx

protected

Definition at line 133 of file segmentation_impl.h.

◆ m_current_slot_no

int32_t deeplima::segmentation::impl::SegmentationImpl::m_current_slot_no

protected

Definition at line 129 of file segmentation_impl.h.

◆ m_current_slot_timepoints

uint32_t deeplima::segmentation::impl::SegmentationImpl::m_current_slot_timepoints

protected

Definition at line 127 of file segmentation_impl.h.

◆ m_current_timepoint

uint64_t deeplima::segmentation::impl::SegmentationImpl::m_current_timepoint

protected

Definition at line 126 of file segmentation_impl.h.

◆ m_decoder

OutputDecoder deeplima::segmentation::impl::SegmentationImpl::m_decoder

protected

Definition at line 124 of file segmentation_impl.h.

◆ m_input_encoder

InputEncoder deeplima::segmentation::impl::SegmentationImpl::m_input_encoder

protected

Definition at line 123 of file segmentation_impl.h.

◆ m_last_completed_slot

int32_t deeplima::segmentation::impl::SegmentationImpl::m_last_completed_slot

protected

Definition at line 130 of file segmentation_impl.h.

The documentation for this class was generated from the following files:

deeplima/include/deeplima/segmentation/impl/segmentation_impl.h
deeplima/libs/tasks/segmentation/inference/segmentation_impl.cpp

Public Member Functions

Protected Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ SegmentationImpl()

◆ ~SegmentationImpl()

Member Function Documentation

◆ acquire_slot()

◆ finalize()

◆ handle_timepoint()

◆ increment_timepoint()

◆ init()

◆ load()

◆ no_more_data()

◆ parse_from_stream()

◆ predicts_mwt()

◆ register_handler()

◆ send_next_results()

◆ send_results()

◆ vectorize_timepoint()

Member Data Documentation

◆ m_buff_set

◆ m_char_len

◆ m_curr_buff_idx

◆ m_current_slot_no

◆ m_current_slot_timepoints

◆ m_current_timepoint

◆ m_decoder

◆ m_input_encoder

◆ m_last_completed_slot