LIMA
Libre Multilingual Analyzer — C++ API
Loading...
Searching...
No Matches
ltrToken.cpp
Go to the documentation of this file.
1// Copyright 2002-2013 CEA LIST
2// SPDX-FileCopyrightText: 2022 CEA LIST <gael.de-chalendar@cea.fr>
3//
4// SPDX-License-Identifier: MIT
5
20#include "ltrToken.h"
21
24
25
26using namespace Lima::Common::Misc;
27using namespace std;
28
29namespace Lima {
30namespace Common {
31namespace BagOfWords {
32
33
38// -----------------------------------------------------------------------------
39// -- destructor
40// -----------------------------------------------------------------------------
41
44
45
46// -----------------------------------------------------------------------------
47// -- copying
48// -----------------------------------------------------------------------------
49
51vector<pair<boost::shared_ptr< BoWToken >, bool> >() {
52 for (LTR_Token::const_iterator itBow = tok.begin();
53 itBow != tok.end(); itBow ++) {
54 this->push_back(
55 make_pair( boost::shared_ptr< BoWToken >(new BoWToken(*(itBow->first))),
56 itBow->second)
57 );
58 }
59}
60
61
62// -----------------------------------------------------------------------------
63// -- reading/writing
64// -----------------------------------------------------------------------------
65
66void LTR_Token::binaryWriteOn(std::ostream& os) const {
67
68 BoWBinaryWriter writer;
69 writeCodedInt(os, this->size());
70 for (LTR_Token::const_iterator itTok = this->begin();
71 itTok != this->end(); itTok ++) {
72 writer.writeSimpleToken(os, itTok->first);
73 if (itTok->second) {
74 writeCodedInt(os, 1);
75 }
76 else {
77 writeCodedInt(os, 0);
78 }
79 }
80}
81
82void LTR_Token::binaryReadFrom(std::istream& is) {
83
84 BoWBinaryReader reader;
85 uint64_t tokenNb = readCodedInt(is);
86 for (uint64_t i = 1; i <= tokenNb; i ++) {
87 boost::shared_ptr< BoWToken > bowTok(new BoWToken());
88 reader.readSimpleToken(is, bowTok);
89 uint64_t selectionFlag = readCodedInt(is);
90 this->push_back(make_pair(bowTok, selectionFlag != 0));
91 }
92}
93
94
95// -----------------------------------------------------------------------------
96// -- printing
97// -----------------------------------------------------------------------------
98
99std::ostream& operator<<(std::ostream& os, const LTR_Token& tok)
100{
101 bool first=true;
102 for (const auto& t: tok) {
103 if (first) { first=false; } else { os << " | "; }
104 if (t.first==nullptr) { os << "NULL"; } else { os << *(t.first); }
105 if (t.second) os << "*";
106 }
107 return os;
108}
109
110QDebug& operator<<(QDebug& os, const LTR_Token& tok)
111{
112 bool first=true;
113 for (const auto& t: tok) {
114 if (first) { first=false; } else { os << " | "; }
115 if (t.first==nullptr) { os << "NULL"; } else { os << *(t.first); }
116 if (t.second) os << "*";
117 }
118 return os;
119}
120
121
122} // BagOfWords
123} // Common
124} // Lima
void readSimpleToken(std::istream &file, boost::shared_ptr< BoWToken > token)
void writeSimpleToken(std::ostream &file, const boost::shared_ptr< BoWToken > token) const
This class contains the representation of an element of the bag of words.
Definition bowToken.h:46
a token of a linear text representation Each LTR_Token represents the set of words that start at a po...
Definition ltrToken.h:48
void binaryReadFrom(std::istream &is)
Definition ltrToken.cpp:82
void binaryWriteOn(std::ostream &os) const
Definition ltrToken.cpp:66
=======================================================================
T & operator<<(T &qd, const BoWType &bt)
uint64_t readCodedInt(std::istream &file)
read a integer coded in variable-byte format in a file
void writeCodedInt(std::ostream &file, const uint64_t number)
write a integer coded in variable-byte format in a file
NAUTITIA.
STL namespace.