15#ifndef FSA_COMPOUNDS_STRING_ACCESS_H
16#define FSA_COMPOUNDS_STRING_ACCESS_H
30 bool operator()(
const std::pair<uint64_t,uint64_t>& p1,
const std::pair<uint64_t,uint64_t>& p2)
const
32 if( p1.first < p2.first )
34 else if( p1.first > p2.first )
37 return ( p1.second < p2.second);
55 int64_t
getIndex(
const std::vector<Lima::LimaString > & compounds )
const;
57 std::vector<Lima::LimaString > & compounds )
const;
58 void read (
const std::string & filename );
59 void read ( std::istream & is );
60 void write(
const std::string & filename );
61 void write( std::ostream & os );
62 void readSimple (
const std::string& filename );
68 void addWord (
const std::vector<Lima::LimaString > & compounds );
72 int64_t findId3( uint64_t tId1, uint64_t tId2 )
const;
74 int64_t findPos( uint64_t tId1, uint64_t tId2, uint64_t& pos )
const;
78 SimpleAccessByString m_simpleAccessByString;
81 typedef std::map<std::pair<uint64_t,uint64_t>, uint64_t,
ltTermIdPair> BuilderCompoundsSet;
82 BuilderCompoundsSet m_builderCompoundsSet;
85 typedef std::vector<CompoundsComposition> ReaderCompoundsSet;
86 ReaderCompoundsSet m_readerCompoundsSet;
88 std::vector<uint64_t> m_compoundsDecomposition;
91template <
typename SimpleAccessByString>
93 : m_simpleAccessByString(trie_direction_fwd), m_nextTid(
MAX_SIMPLE_TID)
97 LDEBUG <<
"CompoundStringAccess::CompoundStringAccess()";
101template <
typename SimpleAccessByString>
106template <
typename SimpleAccessByString>
108 return( m_simpleAccessByString.getSize() + m_readerCompoundsSet.size() );
111template <
typename SimpleAccessByString>
113 const std::vector<Lima::LimaString > & compounds )
const {
117 LDEBUG <<
"CompoundStringAccess::getIndex(";
118 std::vector<Lima::LimaString >::const_iterator forwardCompoundsIt = compounds.begin();
119 if( forwardCompoundsIt != compounds.end() ) {
122 forwardCompoundsIt++;
124 for( ; forwardCompoundsIt != compounds.end() ; forwardCompoundsIt++ ) {
131 std::vector<Lima::LimaString >::const_reverse_iterator compoundsIt =
133 if( compoundsIt == compounds.rend() ) {
137 int64_t tId2 = m_simpleAccessByString.getIndex( *compoundsIt );
142 for( ; compoundsIt != compounds.rend(); compoundsIt++ ) {
143 uint64_t tId1 = m_simpleAccessByString.getIndex( *compoundsIt );
148 LDEBUG <<
"CompoundStringAccess::getIndex: findId3(" << tId1 <<
"," << tId2 <<
")= tId2 =";
150 tId2 = findId3(tId1,tId2);
160template <
typename SimpleAccessByString>
165 LDEBUG <<
"CompoundStringAccess::addWord(" << word <<
")";
167 m_simpleAccessByString.addWord(word);
170template <
typename SimpleAccessByString>
172 const std::vector<Lima::LimaString > & compounds ) {
175 LDEBUG <<
"CompoundStringAccess::addWord(";
176 std::vector<Lima::LimaString >::const_iterator forwardCompoundsIt = compounds.begin();
177 if( forwardCompoundsIt != compounds.end() ) {
180 forwardCompoundsIt++;
182 for( ; forwardCompoundsIt != compounds.end() ; forwardCompoundsIt++ ) {
190 std::vector<Lima::LimaString >::const_reverse_iterator compoundsIt =
192 if( compoundsIt == compounds.rend() ) {
195 uint64_t tId2 = m_simpleAccessByString.getIndex( *compoundsIt );
199 LERROR <<
"CompoundStringAccess::addWord: " << text8
200 <<
" is not in m_simpleAccessByString!";
204 for( ; compoundsIt != compounds.rend() ; compoundsIt++ ) {
205 uint64_t tId1 = m_simpleAccessByString.getIndex( *compoundsIt );
212 BuilderCompoundsSet::iterator pos = m_builderCompoundsSet.find(
213 std::make_pair(tId1, tId2) );
215 if( pos != m_builderCompoundsSet.end() ) {
216 tId3 = (*pos).second;
221 LDEBUG <<
"CompoundStringAccess::addWord: insert( (" << tId1
222 <<
"," << tId2 <<
") , " << tId3 <<
")";
225 m_builderCompoundsSet.insert( std::make_pair ( std::make_pair(tId1, tId2), tId3 ) );
232template <
typename SimpleAccessByString>
234 uint64_t tId1, uint64_t tId2 )
const {
236 return findPos( tId1, tId2, pos );
269template <
typename SimpleAccessByString>
270int64_t CompoundStringAccess<SimpleAccessByString>::findPos(
271 uint64_t tId1, uint64_t tId2, uint64_t& pos )
const {
274 LDEBUG <<
"CompoundStringAccess::findPos(" << tId1 <<
"," << tId2 <<
")";
278 uint64_t max = m_readerCompoundsSet.size();
280 for( ; (max - min) > 1 ; ) {
281 uint64_t range = (max - min)/2;
283 if( composition.cmp1Id < tId1 ) {
285 LDEBUG <<
"CompoundStringAccess::findPos: comp.cmp1Id < tId1, range = " << range
286 <<
"," << composition.
cmp1Id <<
"<" << tId1 <<
": min <- min + range";
290 else if( composition.cmp1Id > tId1 ) {
292 LDEBUG <<
"CompoundStringAccess::findPos: compcmp1Id > tId1, range = " << range
293 <<
"," << composition.cmp1Id <<
">" << tId1 <<
": max <- min + range";
297 else if( composition.cmp2Id < tId2 ) {
299 LDEBUG <<
"CompoundStringAccess::findPos: comp.cmp2Id < tId2 , range = " << range
300 <<
"," << composition.cmp2Id <<
"<" << tId2 <<
": min <- min + range";
304 else if( composition.cmp2Id > tId2 ) {
306 LDEBUG <<
"CompoundStringAccess::findPos: comp.cmp2Id > tId2 , range = " << range
307 <<
"," << composition.cmp2Id <<
">" << tId2 <<
": max <- min + range";
313 LDEBUG <<
"CompoundStringAccess::findPos: else , range = " << range
314 <<
"," <<
"return: tId3 = " << composition.tId3;
317 return composition.tId3;
325 if( composition.cmp1Id == tId1 ) {
326 if( composition.cmp2Id == tId2 ) {
328 LDEBUG <<
"CompoundStringAccess::findPos: comp.cmp1Id == tId1, comp.cmp2Id == tId2"
329 <<
", return: tId3 = " << composition.
tId3;
332 return composition.tId3;
334 else if( composition.cmp2Id < tId1 ) {
337 else if( composition.cmp2Id > tId2 ) {
341 else if( composition.cmp1Id > tId1 ) {
348 LDEBUG <<
"CompoundStringAccess::findPos: comp.cmp1Id != tId1 || comp.cmp2Id != tId2"
354template <
typename SimpleAccessByString>
356 std::vector<Lima::LimaString > & compound )
const {
359 LDEBUG <<
"CompoundStringAccess::getSpelling(" << index <<
")";
362 if( (index -
MAX_SIMPLE_TID) >= m_compoundsDecomposition.size() ) {
363 std::ostringstream ostr;
364 ostr <<
"ERROR! CompoundStringAccess::getSpelling: index (" << index
366 <<
") >= m_compoundsDecomposition.size() (" << m_compoundsDecomposition.size() <<
")";
370 uint64_t offset = m_compoundsDecomposition[index -
MAX_SIMPLE_TID];
372 LDEBUG <<
"CompoundStringAccess::getSpelling: offset=" << offset;
376 LDEBUG <<
"CompoundStringAccess::getSpelling: Composition={("
377 << composition.
tId3 <<
") -> "
380 assert(composition.
tId3 == index);
381 index = composition.
cmp2Id;
382 compound.push_back(m_simpleAccessByString.getSpelling(composition.
cmp1Id));
384 if( index >= m_simpleAccessByString.getSize() ) {
385 std::ostringstream ostr;
386 ostr <<
"ERROR! CompoundStringAccess::getSpelling: index (" << index
387 <<
") >= m_simpleAccessByString.getSize() (" << m_simpleAccessByString.getSize() <<
")";
390 compound.push_back(m_simpleAccessByString.getSpelling(index));
393template <
typename SimpleAccessByString>
395 const std::string & filename )
399 LDEBUG <<
"CompoundStringAccess::readSimple(" << filename <<
")";
401 m_simpleAccessByString.read(filename);
405template <
typename SimpleAccessByString>
407 const std::string & filename )
411 LDEBUG <<
"CompoundStringAccess::read(" << filename <<
")";
413 std::ifstream is(filename.c_str(), std::ios::binary );
415 std::string mess =
"CompoundStringAccess::read: Can't open file " + filename;
425template <
typename SimpleAccessByString>
431 LDEBUG <<
"CompoundStringAccess::read()";
434 m_simpleAccessByString.read( is );
436 readCompoundBody( is );
439template <
typename SimpleAccessByString>
477 uint64_t nbCompounds;
478 is.read( (
char *)(&(nbCompounds )),
sizeof(nbCompounds ) );
480 LDEBUG <<
"CompoundStringAccess::readCompoundBody: nbCompounds="
486 LDEBUG <<
"CompoundStringAccess::readCompoundBody: reserve memory for m_readerCompoundsSet "
489 m_readerCompoundsSet.reserve(nbCompounds);
491 LDEBUG <<
"CompoundStringAccess::readCompoundBody: resize m_compoundsDecomposition to "
494 m_compoundsDecomposition.resize(nbCompounds);
498 for( uint64_t n = 0 ; n < nbCompounds ; n++ ) {
500 is.read( (
char *)(&(newElement.
cmp1Id)),
sizeof(newElement.
cmp1Id) );
501 is.read( (
char *)(&(newElement.
cmp2Id)),
sizeof(newElement.
cmp2Id) );
502 is.read( (
char *)(&(newElement.
tId3)),
sizeof(newElement.
tId3) );
504 LDEBUG <<
"CompoundStringAccess::readCompoundBody: Composition= {"
505 << newElement.
cmp1Id <<
"," << newElement.
cmp2Id <<
"," << newElement.
tId3<<
"}";
507 m_readerCompoundsSet.push_back(newElement);
510 LDEBUG <<
"CompoundStringAccess::readCompoundBody: m_compoundsDecomposition["
519template <
typename SimpleAccessByString>
523 std::ofstream os(filename.c_str(), std::ios::out | std::ios::binary | std::ios::app );
525 std::string mess =
"CompoundStringAccess::write: Can't open file " + filename;
537template <
typename SimpleAccessByString>
541 writeCompoundBody( os );
544template <
typename SimpleAccessByString>
550 LDEBUG <<
"CompoundStringAccess::writeCompoundBody()";
553 uint64_t nbCompounds = m_builderCompoundsSet.size();
555 LDEBUG <<
"CompoundStringAccess::writeCompoundBody: write nbCompounds = " << nbCompounds;
557 os.write( (
const char*)(&nbCompounds),
sizeof(nbCompounds) );
560 BuilderCompoundsSet::const_iterator compIt = m_builderCompoundsSet.begin();
561 for( ; compIt != m_builderCompoundsSet.end() ; compIt++ ) {
562 uint64_t cmp1Id = (*compIt).first.first;
563 uint64_t cmp2Id = (*compIt).first.second;
564 uint64_t tId3 = (*compIt).second;
566 LDEBUG <<
"CompoundStringAccess::writeCompoundBody: write cmp1Id = " << cmp1Id
567 <<
", cmp2Id = " << cmp2Id
568 <<
", tId3 = " << tId3
571 os.write( (
const char*)(&cmp1Id),
sizeof(cmp1Id) );
572 os.write( (
const char*)(&cmp2Id),
sizeof(cmp2Id) );
573 os.write( (
const char*)(&tId3),
sizeof(tId3) );
581#include "CompoundStringAccess.tcc"
#define COMPSTRACCESSLOGINIT
Use this exception to signal the used of a wrongly initialized LIMA dictionary.
Use this exception to signal a wrong access to a LIMA dictionary.
void writeCompoundBody(std::ostream &os)
void readSimple(const std::string &filename)
void getSpelling(const uint64_t index, std::vector< Lima::LimaString > &compounds) const
void readCompoundBody(std::istream &is)
CompoundStringAccess(bool trie_direction_fwd=true)
void read(const std::string &filename)
int64_t getIndex(const std::vector< Lima::LimaString > &compounds) const
void write(const std::string &filename)
void addWord(const std::vector< Lima::LimaString > &compounds)
virtual ~CompoundStringAccess()
struct Lima::Common::FsaAccess::_compoundsComposition CompoundsComposition
std::string limastring2utf8stdstring(const Lima::LimaString &phrase, uint32_t size0)
Convert a wide string to a string , in dest up to size bytes.
bool operator()(const std::pair< uint64_t, uint64_t > &p1, const std::pair< uint64_t, uint64_t > &p2) const