LIMA
Libre Multilingual Analyzer — C++ API
Loading...
Searching...
No Matches
CompoundStringAccess.h
Go to the documentation of this file.
1// Copyright 2002-2019 CEA LIST
2// SPDX-FileCopyrightText: 2022 CEA LIST <gael.de-chalendar@cea.fr>
3//
4// SPDX-License-Identifier: MIT
5
6/***************************************************************************
7 * Copyright (C) 2003 by CEA *
8 * author Olivier MESNARD olivier.mesnard@cea.fr *
9 * *
10 * CompoundStringAccess.h - description *
11 * AccessByString optimized for lexicon with coupounds terms *
12 * based on FsaAccesSpare16 for simple words and *
13 * std::map<pair<id1,id2>,id3> for compounds. *
14 ***************************************************************************/
15#ifndef FSA_COMPOUNDS_STRING_ACCESS_H
16#define FSA_COMPOUNDS_STRING_ACCESS_H
17
18
19#include "common/LimaCommon.h"
20
21namespace Lima {
22namespace Common {
23namespace FsaAccess {
24
25// typedef std::map<std::pair<uint64_t,uint64_t>, uint64_t > BuilderCompoundsSet;
26
27// Structure pour la liste candidats par ordre de taille de contexte d�roissant
29{
30 bool operator()(const std::pair<uint64_t,uint64_t>& p1, const std::pair<uint64_t,uint64_t>& p2) const
31 {
32 if( p1.first < p2.first )
33 return true;
34 else if( p1.first > p2.first )
35 return false;
36 else
37 return ( p1.second < p2.second);
38 }
39};
40
41
42typedef struct _compoundsComposition {
43 uint64_t cmp1Id;
44 uint64_t cmp2Id;
45 uint64_t tId3;
47
48template <typename SimpleAccessByString> class CompoundStringAccess
49{
50 public:
51 CompoundStringAccess(bool trie_direction_fwd=true);
52 virtual ~CompoundStringAccess();
53
54 uint64_t getSize() const;
55 int64_t getIndex( const std::vector<Lima::LimaString > & compounds ) const;
56 void getSpelling(const uint64_t index,
57 std::vector<Lima::LimaString > & compounds ) const;
58 void read ( const std::string & filename );
59 void read ( std::istream & is );
60 void write( const std::string & filename );
61 void write( std::ostream & os );
62 void readSimple ( const std::string& filename );
63
64 void readCompoundBody( std::istream& is );
65// void writeBody( const std::string & filename, struct stat& sts )
66 void writeCompoundBody( std::ostream& os );
67
68 void addWord ( const std::vector<Lima::LimaString > & compounds );
69 void addWord ( Lima::LimaString & word );
70
71 private:
72 int64_t findId3( uint64_t tId1, uint64_t tId2 ) const;
73 const CompoundsComposition& insertComposition( uint64_t tId1, uint64_t tId2 );
74 int64_t findPos( uint64_t tId1, uint64_t tId2, uint64_t& pos ) const;
75
76// uint64_t m_maxSimpleId;
77
78 SimpleAccessByString m_simpleAccessByString;
79
80 // utilisé par le mode builder pour gérer les mots composés (addWord)
81 typedef std::map<std::pair<uint64_t,uint64_t>, uint64_t, ltTermIdPair> BuilderCompoundsSet;
82 BuilderCompoundsSet m_builderCompoundsSet;
83 uint64_t m_nextTid;
84 // utilisé par le mode reader pour gérer les mots composés
85 typedef std::vector<CompoundsComposition> ReaderCompoundsSet;
86 ReaderCompoundsSet m_readerCompoundsSet;
87 // utilisé par le mode reader pour la fonction getSpelling
88 std::vector<uint64_t> m_compoundsDecomposition;
89};
90
91template <typename SimpleAccessByString>
93 : m_simpleAccessByString(trie_direction_fwd), m_nextTid(MAX_SIMPLE_TID)
94{
95#ifdef DEBUG_CD
97 LDEBUG << "CompoundStringAccess::CompoundStringAccess()";
98#endif
99}
100
101template <typename SimpleAccessByString>
105
106template <typename SimpleAccessByString>
108 return( m_simpleAccessByString.getSize() + m_readerCompoundsSet.size() );
109}
110
111template <typename SimpleAccessByString>
113 const std::vector<Lima::LimaString > & compounds ) const {
114
115#ifdef DEBUG_CD
117 LDEBUG << "CompoundStringAccess::getIndex(";
118 std::vector<Lima::LimaString >::const_iterator forwardCompoundsIt = compounds.begin();
119 if( forwardCompoundsIt != compounds.end() ) {
120 std::string text8 = Lima::Common::Misc::limastring2utf8stdstring(*forwardCompoundsIt);
121 LDEBUG << text8;
122 forwardCompoundsIt++;
123 }
124 for( ; forwardCompoundsIt != compounds.end() ; forwardCompoundsIt++ ) {
125 std::string text8 = Lima::Common::Misc::limastring2utf8stdstring(*forwardCompoundsIt);
126 LDEBUG << text8;
127 }
128 LDEBUG << ")";
129#endif
130 // Iterateur pour parcourir du dernier au premier les composants
131 std::vector<Lima::LimaString >::const_reverse_iterator compoundsIt =
132 compounds.rbegin();
133 if( compoundsIt == compounds.rend() ) {
134 return -1;
135 }
136 // index du premier composant;
137 int64_t tId2 = m_simpleAccessByString.getIndex( *compoundsIt );
138#ifdef DEBUG_CD
139 LDEBUG << "CompoundStringAccess::getIndex: m_simple.getIndex(" << Lima::Common::Misc::limastring2utf8stdstring(*compoundsIt) << ")= tId2 =" << tId2;
140#endif
141 compoundsIt++;
142 for( ; compoundsIt != compounds.rend(); compoundsIt++ ) {
143 uint64_t tId1 = m_simpleAccessByString.getIndex( *compoundsIt );
144#ifdef DEBUG_CD
145 LDEBUG << "CompoundStringAccess::getIndex: m_simple.getIndex(" << Lima::Common::Misc::limastring2utf8stdstring(*compoundsIt) << ")= tId1 =" << tId1;
146#endif
147#ifdef DEBUG_CD
148 LDEBUG << "CompoundStringAccess::getIndex: findId3(" << tId1 << "," << tId2 << ")= tId2 =";
149#endif
150 tId2 = findId3(tId1,tId2);
151#ifdef DEBUG_CD
152 LDEBUG << tId2;
153#endif
154 if( tId2 == -1 )
155 return -1;
156 }
157 return tId2;
158}
159
160template <typename SimpleAccessByString>
162 Lima::LimaString & word ) {
163#ifdef DEBUG_CD
165 LDEBUG << "CompoundStringAccess::addWord(" << word << ")";
166#endif
167 m_simpleAccessByString.addWord(word);
168}
169
170template <typename SimpleAccessByString>
172 const std::vector<Lima::LimaString > & compounds ) {
174#ifdef DEBUG_CD
175 LDEBUG << "CompoundStringAccess::addWord(";
176 std::vector<Lima::LimaString >::const_iterator forwardCompoundsIt = compounds.begin();
177 if( forwardCompoundsIt != compounds.end() ) {
178 std::string text8 = Lima::Common::Misc::limastring2utf8stdstring(*forwardCompoundsIt);
179 LDEBUG << text8;
180 forwardCompoundsIt++;
181 }
182 for( ; forwardCompoundsIt != compounds.end() ; forwardCompoundsIt++ ) {
183 std::string text8 = Lima::Common::Misc::limastring2utf8stdstring(*forwardCompoundsIt);
184 LDEBUG << text8;
185 }
186 LDEBUG << ")";
187#endif
188
189 // Iterateur pour parcourir du dernier au premier les composants
190 std::vector<Lima::LimaString >::const_reverse_iterator compoundsIt =
191 compounds.rbegin();
192 if( compoundsIt == compounds.rend() ) {
193 return;
194 }
195 uint64_t tId2 = m_simpleAccessByString.getIndex( *compoundsIt );
196 if( tId2 == 0 ) {
197 std::string text8 = Lima::Common::Misc::limastring2utf8stdstring(*compoundsIt);
198#ifdef DEBUG_CD
199 LERROR << "CompoundStringAccess::addWord: " << text8
200 << " is not in m_simpleAccessByString!";
201#endif
202 }
203 compoundsIt++;
204 for( ; compoundsIt != compounds.rend() ; compoundsIt++ ) {
205 uint64_t tId1 = m_simpleAccessByString.getIndex( *compoundsIt );
206 if( tId1 == 0 ) {
207 std::string text8 = Lima::Common::Misc::limastring2utf8stdstring(*compoundsIt);
208 LERROR << text8;
209 return;
210 }
211// const CompoundsComposition& composition = insertComposition( tId1, tId2 );
212 BuilderCompoundsSet::iterator pos = m_builderCompoundsSet.find(
213 std::make_pair(tId1, tId2) );
214 uint64_t tId3;
215 if( pos != m_builderCompoundsSet.end() ) {
216 tId3 = (*pos).second;
217 }
218 else {
219 tId3 = m_nextTid++;
220#ifdef DEBUG_CD
221 LDEBUG << "CompoundStringAccess::addWord: insert( (" << tId1
222 << "," << tId2 << ") , " << tId3 << ")";
223#endif
224// m_builderCompoundsSet.insert(make_pair(make_pair(tId1, tId2), tId3) );
225 m_builderCompoundsSet.insert( std::make_pair ( std::make_pair(tId1, tId2), tId3 ) );
226 }
227 tId2 = tId3;
228 }
229}
230
231// recherche dichotomique de tid3
232template <typename SimpleAccessByString>
234 uint64_t tId1, uint64_t tId2 ) const {
235 uint64_t pos;
236 return findPos( tId1, tId2, pos );
237}
238
239/*
240// insertion d'un nouveau composant
241template <typename SimpleAccessByString>
242const CompoundsComposition& CompoundStringAccess<SimpleAccessByString>::insertComposition(
243 uint64_t tId1, uint64_t tId2 ) {
244#ifdef DEBUG_CD
245 COMPSTRACCESSLOGINIT;
246 LDEBUG << "CompoundStringAccess::insertComposition(" << tId1 << "," << tId2 << ")";
247#endif
248
249 uint64_t pos;
250 uint64_t tId3 = findPos( tId1, tId2, pos );
251#ifdef DEBUG_CD
252 LDEBUG << "CompoundStringAccess::insertComposition: tId3=" << tId3
253 << ", pos = " << pos << ")";
254#endif
255
256 m_compoundsDecomposition.size();
257
258 m_compoundsDecomposition.push_back(pos);
259 CompoundsComposition composition;
260 composition.cmp1Id = tId1;
261 composition.cmp2Id = tId2;
262 composition.tId3 = m_maxSimpleId + m_compoundsDecomposition.size();
263 CompoundsSet::iterator newCompPos = m_compoundsSet.insert( m_compoundsSet.begin() + pos, composition);
264 return *newCompPos;
265}
266*/
267
268// recherche dichotomique de tid3
269template <typename SimpleAccessByString>
270int64_t CompoundStringAccess<SimpleAccessByString>::findPos(
271 uint64_t tId1, uint64_t tId2, uint64_t& pos ) const {
272#ifdef DEBUG_CD
274 LDEBUG << "CompoundStringAccess::findPos(" << tId1 << "," << tId2 << ")";
275#endif
276
277 uint64_t min = 0;
278 uint64_t max = m_readerCompoundsSet.size();
279 // test au milieu du segment [min,max[
280 for( ; (max - min) > 1 ; ) {
281 uint64_t range = (max - min)/2;
282 const CompoundsComposition& composition = m_readerCompoundsSet[min+range];
283 if( composition.cmp1Id < tId1 ) {
284#ifdef DEBUG_CD
285 LDEBUG << "CompoundStringAccess::findPos: comp.cmp1Id < tId1, range = " << range
286 << "," << composition.cmp1Id << "<" << tId1 << ": min <- min + range";
287#endif
288 min = min + range;
289 }
290 else if( composition.cmp1Id > tId1 ) {
291#ifdef DEBUG_CD
292 LDEBUG << "CompoundStringAccess::findPos: compcmp1Id > tId1, range = " << range
293 << "," << composition.cmp1Id << ">" << tId1 << ": max <- min + range";
294#endif
295 max = min + range;
296 }
297 else if( composition.cmp2Id < tId2 ) {
298#ifdef DEBUG_CD
299 LDEBUG << "CompoundStringAccess::findPos: comp.cmp2Id < tId2 , range = " << range
300 << "," << composition.cmp2Id << "<" << tId2 << ": min <- min + range";
301#endif
302 min = min + range;
303 }
304 else if( composition.cmp2Id > tId2 ) {
305#ifdef DEBUG_CD
306 LDEBUG << "CompoundStringAccess::findPos: comp.cmp2Id > tId2 , range = " << range
307 << "," << composition.cmp2Id << ">" << tId2 << ": max <- min + range";
308#endif
309 max = min + range;
310 }
311 else {
312#ifdef DEBUG_CD
313 LDEBUG << "CompoundStringAccess::findPos: else , range = " << range
314 << "," << "return: tId3 = " << composition.tId3;
315#endif
316 pos = min+range;
317 return composition.tId3;
318 }
319 }
320 if(max == min) {
321 pos = min;
322 return -1;
323 }
324 const CompoundsComposition& composition = m_readerCompoundsSet[min];
325 if( composition.cmp1Id == tId1 ) {
326 if( composition.cmp2Id == tId2 ) {
327#ifdef DEBUG_CD
328 LDEBUG << "CompoundStringAccess::findPos: comp.cmp1Id == tId1, comp.cmp2Id == tId2"
329 << ", return: tId3 = " << composition.tId3;
330#endif
331 pos = min;
332 return composition.tId3;
333 }
334 else if( composition.cmp2Id < tId1 ) {
335 pos = max;
336 }
337 else if( composition.cmp2Id > tId2 ) {
338 pos = min;
339 }
340 }
341 else if( composition.cmp1Id > tId1 ) {
342 pos = min;
343 }
344 else{
345 pos = max;
346 }
347#ifdef DEBUG_CD
348 LDEBUG << "CompoundStringAccess::findPos: comp.cmp1Id != tId1 || comp.cmp2Id != tId2"
349 << ", return: -1!";
350#endif
351 return -1;
352}
353
354template <typename SimpleAccessByString>
356 std::vector<Lima::LimaString > & compound ) const {
357#ifdef DEBUG_CD
359 LDEBUG << "CompoundStringAccess::getSpelling(" << index << ")";
360#endif
361 while( index >= MAX_SIMPLE_TID ) {
362 if( (index - MAX_SIMPLE_TID) >= m_compoundsDecomposition.size() ) {
363 std::ostringstream ostr;
364 ostr << "ERROR! CompoundStringAccess::getSpelling: index (" << index
365 << ") - MAX_SIMPLE_TID (" << MAX_SIMPLE_TID
366 << ") >= m_compoundsDecomposition.size() (" << m_compoundsDecomposition.size() << ")";
367 throw( AccessByStringOutOfRange(ostr.str()) );
368 }
369
370 uint64_t offset = m_compoundsDecomposition[index - MAX_SIMPLE_TID];
371#ifdef DEBUG_CD
372 LDEBUG << "CompoundStringAccess::getSpelling: offset=" << offset;
373#endif
374 const CompoundsComposition& composition = m_readerCompoundsSet[offset];
375#ifdef DEBUG_CD
376 LDEBUG << "CompoundStringAccess::getSpelling: Composition={("
377 << composition.tId3 << ") -> "
378 << composition.cmp1Id << "," << composition.cmp2Id;
379#endif
380 assert(composition.tId3 == index);
381 index = composition.cmp2Id;
382 compound.push_back(m_simpleAccessByString.getSpelling(composition.cmp1Id));
383 }
384 if( index >= m_simpleAccessByString.getSize() ) {
385 std::ostringstream ostr;
386 ostr << "ERROR! CompoundStringAccess::getSpelling: index (" << index
387 << ") >= m_simpleAccessByString.getSize() (" << m_simpleAccessByString.getSize() << ")";
388 throw( AccessByStringOutOfRange(ostr.str()) );
389 }
390 compound.push_back(m_simpleAccessByString.getSpelling(index));
391}
392
393template <typename SimpleAccessByString>
395 const std::string & filename )
396{
397#ifdef DEBUG_CD
399 LDEBUG << "CompoundStringAccess::readSimple(" << filename << ")";
400#endif
401 m_simpleAccessByString.read(filename);
402
403}
404
405template <typename SimpleAccessByString>
407 const std::string & filename )
408{
409#ifdef DEBUG_CD
411 LDEBUG << "CompoundStringAccess::read(" << filename << ")";
412#endif
413 std::ifstream is(filename.c_str(), std::ios::binary );
414 if( is.bad() ) {
415 std::string mess = "CompoundStringAccess::read: Can't open file " + filename;
416#ifdef DEBUG_CD
417 LERROR << mess;
418#endif
419 throw( AccessByStringNotInitialized( mess ) );
420 }
421
423}
424
425template <typename SimpleAccessByString>
427 std::istream& is )
428{
429#ifdef DEBUG_CD
431 LDEBUG << "CompoundStringAccess::read()";
432#endif
433
434 m_simpleAccessByString.read( is );
435
436 readCompoundBody( is );
437}
438
439template <typename SimpleAccessByString>
441 std::istream& is )
442{
443#ifdef DEBUG_CD
445#endif
446
447/* uint64_t compoundsDataStreampos = m_simpleAccessByString.FsaAccessHeader::getStreamPos();
448#ifdef DEBUG_CD
449 LDEBUG << "CompoundStringAccess::readCompoundBody(" << filename << "):"
450 << " compoundsDataStreampos=" << compoundsDataStreampos;
451#endif
452*/
453 // offset pour atteindre les mots compos
454// uint64_t compoundsDataOffset =
455// m_simpleAccessByString.FsaAccessHeader::getCompoundsDataOffset();
456 // nombre de mots composes pour reserver la memoire
457// uint64_t nbCompounds =
458// m_simpleAccessByString.FsaAccessHeader::getNbCompounds();
459
460/*
461 std::ifstream is(filename.c_str(), std::ios::binary );
462 if( is.bad() ) {
463 std::string mess = "CompoundStringAccess::readCompoundBody: Can't open file " + filename;
464 LERROR << mess;
465 throw( AccessByStringNotInitialized( mess ) );
466 }
467
468 // positionnement du fichier sur la zone de donnees des mots composes
469#ifdef DEBUG_CD
470 LDEBUG << "CompoundStringAccess::readCompoundBody: seek to "
471 << compoundsDataStreampos;
472#endif
473 is.seekg(compoundsDataStreampos,std::ios::beg) ;
474*/
475
476 //lecture du nombre de mots composés
477 uint64_t nbCompounds;
478 is.read( (char *)(&(nbCompounds )), sizeof(nbCompounds ) );
479#ifdef DEBUG_CD
480 LDEBUG << "CompoundStringAccess::readCompoundBody: nbCompounds="
481 << nbCompounds;
482#endif
483
484 // reservation memoire
485#ifdef DEBUG_CD
486 LDEBUG << "CompoundStringAccess::readCompoundBody: reserve memory for m_readerCompoundsSet "
487 ;
488#endif
489 m_readerCompoundsSet.reserve(nbCompounds);
490#ifdef DEBUG_CD
491 LDEBUG << "CompoundStringAccess::readCompoundBody: resize m_compoundsDecomposition to "
492 << nbCompounds;
493#endif
494 m_compoundsDecomposition.resize(nbCompounds);
495
496 // lecture de la composition des mots compose
497// uint64_t size = m_simpleAccessByString.getSize();
498 for( uint64_t n = 0 ; n < nbCompounds ; n++ ) {
499 CompoundsComposition newElement;
500 is.read( (char *)(&(newElement.cmp1Id)), sizeof(newElement.cmp1Id) );
501 is.read( (char *)(&(newElement.cmp2Id)), sizeof(newElement.cmp2Id) );
502 is.read( (char *)(&(newElement.tId3)), sizeof(newElement.tId3) );
503#ifdef DEBUG_CD
504 LDEBUG << "CompoundStringAccess::readCompoundBody: Composition= {"
505 << newElement.cmp1Id << "," << newElement.cmp2Id << "," << newElement.tId3<< "}";
506#endif
507 m_readerCompoundsSet.push_back(newElement);
508// assert(newElement.tId3 - m_maxSimpleId < nbCompounds);
509#ifdef DEBUG_CD
510 LDEBUG << "CompoundStringAccess::readCompoundBody: m_compoundsDecomposition["
511 << newElement.tId3-MAX_SIMPLE_TID << "]="
512 << n;
513#endif
514 m_compoundsDecomposition[newElement.tId3-MAX_SIMPLE_TID] = n;
515 }
516
517}
518
519template <typename SimpleAccessByString>
520void CompoundStringAccess<SimpleAccessByString>::write( const std::string & filename )
521{
522
523 std::ofstream os(filename.c_str(), std::ios::out | std::ios::binary | std::ios::app );
524 if( os.bad() ) {
525 std::string mess = "CompoundStringAccess::write: Can't open file " + filename;
526#ifdef DEBUG_CD
528 LERROR;
529#endif
530 throw( FsaNotSaved( mess ) );
531 }
532// os.seekp(HEADER_SIZE ,std::ios_base::beg );
534
535}
536
537template <typename SimpleAccessByString>
539
540 //m_simpleAccessByString.write( os );
541 writeCompoundBody( os );
542}
543
544template <typename SimpleAccessByString>
546 std::ostream& os ) {
547
548#ifdef DEBUG_CD
550 LDEBUG << "CompoundStringAccess::writeCompoundBody()";
551#endif
552
553 uint64_t nbCompounds = m_builderCompoundsSet.size();
554#ifdef DEBUG_CD
555 LDEBUG << "CompoundStringAccess::writeCompoundBody: write nbCompounds = " << nbCompounds;
556#endif
557 os.write( (const char*)(&nbCompounds), sizeof(nbCompounds) );
558
559 // sauvegarde de la composition des mots compose
560 BuilderCompoundsSet::const_iterator compIt = m_builderCompoundsSet.begin();
561 for( ; compIt != m_builderCompoundsSet.end() ; compIt++ ) {
562 uint64_t cmp1Id = (*compIt).first.first;
563 uint64_t cmp2Id = (*compIt).first.second;
564 uint64_t tId3 = (*compIt).second;
565#ifdef DEBUG_CD
566 LDEBUG << "CompoundStringAccess::writeCompoundBody: write cmp1Id = " << cmp1Id
567 << ", cmp2Id = " << cmp2Id
568 << ", tId3 = " << tId3
569 ;
570#endif
571 os.write( (const char*)(&cmp1Id), sizeof(cmp1Id) );
572 os.write( (const char*)(&cmp2Id), sizeof(cmp2Id) );
573 os.write( (const char*)(&tId3), sizeof(tId3) );
574 }
575}
576
577
578} // namespace compactDict
579} // namespace Common
580} // namespace Lima
581#include "CompoundStringAccess.tcc"
582
583#endif //FSA_COMPOUNDS_STRING_ACCESS_H
#define MAX_SIMPLE_TID
#define COMPSTRACCESSLOGINIT
Definition LimaCommon.h:203
#define LDEBUG
Definition LimaCommon.h:157
#define LERROR
Definition LimaCommon.h:161
#define FSAAIOLOGINIT
Definition LimaCommon.h:205
Use this exception to signal the used of a wrongly initialized LIMA dictionary.
Definition LimaCommon.h:428
Use this exception to signal a wrong access to a LIMA dictionary.
Definition LimaCommon.h:450
void getSpelling(const uint64_t index, std::vector< Lima::LimaString > &compounds) const
int64_t getIndex(const std::vector< Lima::LimaString > &compounds) const
void addWord(const std::vector< Lima::LimaString > &compounds)
struct Lima::Common::FsaAccess::_compoundsComposition CompoundsComposition
std::string limastring2utf8stdstring(const Lima::LimaString &phrase, uint32_t size0)
Convert a wide string to a string , in dest up to size bytes.
NAUTITIA.
QString LimaString
Definition LimaString.h:33
bool operator()(const std::pair< uint64_t, uint64_t > &p1, const std::pair< uint64_t, uint64_t > &p2) const