LIMA
Libre Multilingual Analyzer — C++ API
Loading...
Searching...
No Matches
textFormater.cpp
Go to the documentation of this file.
1// Copyright 2002-2013 CEA LIST
2// SPDX-FileCopyrightText: 2022 CEA LIST <gael.de-chalendar@cea.fr>
3//
4// SPDX-License-Identifier: MIT
5
15#include "common/LimaCommon.h"
16#include "common/Data/strwstrtools.h"
17
18#include "textFormater.h"
19
20#include <iostream>
21#include <sstream>
22#include <stdexcept>
23#include <string>
24#include <limits.h>
25#include <stdlib.h>
26
27#define LILOGINIT LOGINIT("MediaIdent");
28
29namespace Lima {
30namespace Common {
31namespace Misc {
32
36
40
55int TextFormater::untaggingWithSpaces(std::ostream& status, std::istream& input,
56 std::ostream& output, bool wide,
57 const std::string& endSentenceTag) const
58{
59 LIMA_UNUSED(wide);
60 LILOGINIT;
61 setlocale(LC_CTYPE,"fr_FR.UTF-8");
62
63 size_t nb = 0; // the number of chars in the current tag when we cannot know
64 // if it is bond to the following text (tag after a white space)
65 size_t nbNewLines = 0; // the number of newlines in the current tag when we cannot know
66 // if it is bond to the following text (tag after a white space)
67 size_t position=0; // position in the input stream;
68 enum RetVal {SUCCESS, INVALID_OPENING_TAG_CHAR, INVALID_CLOSING_TAG_CHAR,
69 DUPLICATED_OPENING_TAG_CHAR, DUPLICATED_CLOSING_TAG_CHAR,
70 UNCLOSED_OPENING_TAG_CHAR};
71 RetVal retVal = SUCCESS;
72
73 enum Etat {TEXT, DEBCOL, FINCOL, BLANC, DEBBLANC, FINBLANC, BEGENTITY, ENTITY};
74 Etat etat = BLANC;
75
76 char carLu;
77 char carLu2;
78 std::string s;
79
80 std::ostringstream txt;
81 std::ostringstream tag;
82 std::ostringstream tagValue;
83 std::ostringstream entity;
84
85 while (input.good())
86 {
87 input.get(carLu);
88 LDEBUG << carLu;
89 if (input.eof()) continue;
90 switch (etat)
91 {
92 case TEXT:
93 switch (carLu)
94 {
95 case '<':
96 output << txt.str();
97 txt.str("");
98 tag << '_';
99 tag << ' ';
100 tagValue << carLu;
101 LDEBUG << "TEXT-> DEBCOL";
102 etat = DEBCOL;
103 break;
104 case '>':
105 txt << carLu;
106 status << "Invalid '>' character at " << position << std::endl;
107 retVal = INVALID_CLOSING_TAG_CHAR;
108 break;
109 case '&':
110 output << txt.str();
111 txt.str("");
112 entity << carLu;
113 LDEBUG << "TEXT -> BEGENTITY;";
114 etat = BEGENTITY;
115 break;
116 case ' ':case '\t':case '\n':
117 output << txt.str();;
118 output << carLu;
119 txt.str("");
120 LDEBUG << "TEXT-> BLANC";
121 etat = BLANC;
122 break;
123 default:
124 txt << carLu;
125 break;
126 }
127 break;
128 case BEGENTITY:
129 switch (carLu)
130 {
131 case 'A':;case 'B':;case 'C':;case 'D':;case 'E':;case 'F':;case 'G':;
132 case 'H':;case 'I':;case 'J':;case 'K':;case 'L':;case 'M':;case 'N':;
133 case 'O':;case 'P':;case 'Q':;case 'R':;case 'S':;case 'T':;case 'U':;
134 case 'V':;case 'W':;case 'X':;case 'Y':;case 'Z':;
135 case 'a':;case 'b':;case 'c':;case 'd':;case 'e':;case 'f':;case 'g':;
136 case 'h':;case 'i':;case 'j':;case 'k':;case 'l':;case 'm':;case 'n':;
137 case 'o':;case 'p':;case 'q':;case 'r':;case 's':;case 't':;case 'u':;
138 case 'v':;case 'w':;case 'x':;case 'y':;case 'z':
139 entity << carLu;
140 LDEBUG << "BEGENTITY-> ENTITY";
141 etat = ENTITY;
142 break;
143 case '<':
144 output.put('_');
145 output.put(' ');
146 entity.str("");
147 tag << '_';
148 tag << ' ';
149 LDEBUG << "BEGENTITY-> DEBCOL";
150 etat = DEBCOL;
151 break;
152 case ' ':case '\t':case '\n':
153// output.put('_');
154 output.put(' ');
155 output << carLu;
156 entity.str("");
157 LDEBUG << "BEGENTITY-> BLANC";
158 etat = BLANC;
159 break;
160 default:
161// output.put('_');
162 output.put(' ');
163 output << carLu;
164 entity.str("");
165 LDEBUG << "BEGENTITY-> TEXT";
166 etat = TEXT;
167 break;
168 }
169 break;
170 case ENTITY:
171 switch (carLu)
172 {
173 case 'A':;case 'B':;case 'C':;case 'D':;case 'E':;case 'F':;case 'G':;
174 case 'H':;case 'I':;case 'J':;case 'K':;case 'L':;case 'M':;case 'N':;
175 case 'O':;case 'P':;case 'Q':;case 'R':;case 'S':;case 'T':;case 'U':;
176 case 'V':;case 'W':;case 'X':;case 'Y':;case 'Z':;
177 case 'a':;case 'b':;case 'c':;case 'd':;case 'e':;case 'f':;case 'g':;
178 case 'h':;case 'i':;case 'j':;case 'k':;case 'l':;case 'm':;case 'n':;
179 case 'o':;case 'p':;case 'q':;case 'r':;case 's':;case 't':;case 'u':;
180 case 'v':;case 'w':;case 'x':;case 'y':;case 'z':
181 entity << carLu;
182 break;
183 case '<':
184 for (uint64_t i = 0; i < entity.str().size(); i++)
185 {
186// output << '_';
187 output << ' ';
188 }
189 entity.str("");
190// tag << '_';
191 tag << ' ';
192 LDEBUG << "ENTITY-> DEBCOL";
193 etat = DEBCOL;
194 break;
195 case ' ':case '\t':case '\n':
196 for (uint64_t i = 0; i < entity.str().size(); i++)
197 {
198// output << '_';
199 output << ' ';
200 }
201 output << carLu;
202 entity.str("");
203 LDEBUG << "ENTITY-> BLANC";
204 etat = BLANC;
205 break;
206 case ';':
207 for (uint64_t i = 0; i < entity.str().size()+1; i++)
208 {
209// output << '_';
210 output << ' ';
211 }
212 entity.str("");
213 LDEBUG << "ENTITY-> TEXT";
214 etat = TEXT;
215 break;
216 default:
217 for (uint64_t i = 0; i < entity.str().size(); i++)
218 {
219// output << '_';
220 output << ' ';
221 }
222 output << carLu;
223 entity.str("");
224 LDEBUG << "ENTITY-> TEXT";
225 etat = TEXT;
226 break;
227 }
228 break;
229 case DEBCOL:
230 tagValue << carLu;
231 switch (carLu)
232 {
233 case '<':
234// tag << '_';
235 tag << ' ';
236 status << "Invalid '<' character at " << position << std::endl;
237 retVal = DUPLICATED_OPENING_TAG_CHAR;
238 break;
239 case '>':
240// tag << '_';
241 tag << ' ';
242 LDEBUG << "DEBCOL-> FINCOL";
243 etat = FINCOL;
244 break;
245 case ' ':case '\t':case '\n':
246 tag << carLu;
247 break;
248 default:
249 LDEBUG << "Looking at " << carLu;
250 char buf[MB_LEN_MAX];
251 buf[0] = carLu;
252 input.rdbuf()-> sgetn(buf+1, 9);
253 wchar_t mbc;
254 int transRes = mbtowc(&mbc, buf, MB_LEN_MAX);
255 LDEBUG << "transres value is " << transRes;
256 if (transRes > 1)
257 {
258 LDEBUG << "Got a multibyte char inside tag: " << mbc;
259 for (int i = 1; i < transRes; i++)
260 {
261 input.get(carLu);
262 }
263 }
264// tag << '_';
265 tag << ' ';
266 break;
267 }
268 break;
269 case FINCOL:
270 switch (carLu)
271 {
272 case '<':
273// tag << '_';
274 tag << ' ';
275 tagValue << carLu;
276 LDEBUG << "FINCOL-> DEBCOL";
277 etat = DEBCOL;
278 break;
279 case '>':
280// tag << '_';
281 tag << ' ';
282 tagValue << carLu;
283 status << "Invalid '>' character at " << position << std::endl;
284 retVal = DUPLICATED_CLOSING_TAG_CHAR;
285 break;
286 case ' ':case '\t':case '\n':
287 putTag(status, output, wide, endSentenceTag, tag, tagValue);
288 output << carLu;
289 LDEBUG << "FINCOL-> BLANC";
290 etat = BLANC;
291 break;
292 case '&':
293 putTag(status, output, wide, endSentenceTag, tag, tagValue);
294 entity << carLu;
295 LDEBUG << "FINCOL -> BEGENTITY;";
296 etat = BEGENTITY;
297 break;
298 default:
299 putTag(status, output, wide, endSentenceTag, tag, tagValue);
300 txt << carLu;
301 tagValue << carLu;
302 LDEBUG << "FINCOL-> TEXT";
303 etat = TEXT;
304 break;
305 }
306 break;
307 case BLANC:
308 switch (carLu)
309 {
310 case '&':
311 entity << carLu;
312 LDEBUG << "BLANC -> BEGENTITY;";
313 etat = BEGENTITY;
314 break;
315 case '<':
316 nb = 1;
317 nbNewLines = 0;
318 tagValue << carLu;
319 LDEBUG << "BLANC-> DEBBLANC";
320 etat = DEBBLANC;
321 break;
322 case '>':
323 output << '>';
324 status << "Invalid '>' character at " << position << std::endl;
325 retVal = INVALID_CLOSING_TAG_CHAR;
326 LDEBUG << "BLANC-> TEXT";
327 etat = TEXT;
328 break;
329 case ' ':case '\t':case '\n':
330 output << carLu;
331 break;
332 default:
333 txt << carLu;
334 LDEBUG << "BLANC-> TEXT";
335 etat = TEXT;
336 break;
337 }
338 break;
339 case DEBBLANC:
340 tagValue << carLu;
341 switch (carLu)
342 {
343 case '<':
344 nb++;
345 status << "Duplicated '<' character at " << position << std::endl;
346 retVal = DUPLICATED_OPENING_TAG_CHAR;
347 break;
348 case '>':
349 nb++;
350 LDEBUG << "DEBBLANC-> FINBLANC";
351 etat = FINBLANC;
352 break;
353 case '\n':
354 nbNewLines++;
355 break;
356 default:
357 LDEBUG << "Looking at " << carLu;
358 char buf[MB_LEN_MAX];
359 buf[0] = carLu;
360 std::streamsize got = input.rdbuf()-> sgetn(buf+1, 9);
361 for (std::streamsize i = 0; i < got ; i++)
362 input.rdbuf()-> sungetc();
363 for (uint64_t i = 0; i<MB_LEN_MAX; i++)
364 LDEBUG << buf[i];
365 LDEBUG;
366 wchar_t mbc;
367 int transRes = mbtowc(&mbc, buf, MB_LEN_MAX);
368 LDEBUG << "transres is " << transRes;
369 if (transRes > 1)
370 {
371 LDEBUG << "Got a multibyte char inside tag: " << mbc;
372 for (int i = 1; i < transRes; i++)
373 {
374 input.get(carLu);
375 }
376 }
377 nb++;
378 break;
379 }
380 break;
381 case FINBLANC:
382 switch (carLu)
383 {
384 case '<':
385 nb++;
386 tagValue << carLu;
387 LDEBUG << "FINBLANC-> DEBBLANC";
388 etat = DEBBLANC;
389 break;
390 case '>':
391 status << "Duplicated '>' character at " << position << std::endl;
392 retVal = DUPLICATED_CLOSING_TAG_CHAR;
393 putWhites(status, output, wide,endSentenceTag, tagValue,
394 output, carLu, ' ', nb, nbNewLines);
395 LDEBUG << "FINBLANC-> TEXT";
396 etat = TEXT;
397 break;
398 case ' ':case '\t':case '\n':
399 putWhites(status, output, wide,endSentenceTag, tagValue,
400 output, carLu, ' ', nb, nbNewLines);
401 LDEBUG << "FINBLANC-> BLANC";
402 etat = BLANC;
403 break;
404 case '&':
405// putWhites(status, output, wide,endSentenceTag, tagValue,
406// entity, carLu, '_', nb, nbNewLines);
407 putWhites(status, output, wide,endSentenceTag, tagValue,
408 entity, carLu, ' ', nb, nbNewLines);
409 LDEBUG << "FINBLANC -> BEGENTITY;";
410 etat = BEGENTITY;
411 break;
412 default:
413// putWhites(status, output, wide,endSentenceTag, tagValue,
414// txt, carLu, '_', nb, nbNewLines);
415 putWhites(status, output, wide,endSentenceTag, tagValue,
416 txt, carLu, ' ', nb, nbNewLines);
417 LDEBUG << "FINBLANC-> TEXT";
418 etat = TEXT;
419 break;
420 }
421 break;
422 default:
423 throw std::runtime_error((std::string("unknown state %d.\n", int(etat))).c_str());
424 }
425 ++position;
426 }
427 if ( (etat == DEBCOL) || (etat == DEBBLANC) )
428 {
429 status << "Unclosed tag at EOF (" << position << ")" << std::endl;
430 retVal = UNCLOSED_OPENING_TAG_CHAR;
431 if (etat == DEBCOL)
432 {
433 output << tag.str();
434 }
435 else
436 {
437// s = std::string(nb/2, '_');
438 s = std::string(nb/2, ' ');
439 s.append(nbNewLines, '\n');
440// if (nb%2 == 0) s.append(nb/2, '_');
441 if (nb%2 == 0) s.append(nb/2, ' ');
442// else s.append(nb/2 + 1, '_');
443 else s.append(nb/2 + 1, ' ');
444 output << s;
445 nb = 0;
446 nbNewLines = 0;
447 }
448 }
449
450 else if ((etat == FINCOL) || (etat == FINBLANC))
451 {
452 output << tag.str();
453 }
454
455 else if (etat == TEXT)
456 {
457 output << txt.str();
458 }
459 else {} // BLANC nothing to do
460
461 return int(retVal);
462}
463
464void TextFormater::putTag(std::ostream& status, std::ostream& output, bool wide,
465 const std::string& endSentenceTag, std::ostringstream& tag, std::ostringstream& tagValue) const
466{
467 LIMA_UNUSED(status);
468 LIMA_UNUSED(wide);
469 LILOGINIT;
470 std::string stag = tag.str();
471 if ( (endSentenceTag != "")
472 && (tagValue.str().find(endSentenceTag,0) != std::string::npos)
473 && (stag.length() > 0) )
474 stag[0] = '.';
475 output << stag;
476 tag.str("");
477 tagValue.str("");
478
479}
480
481void TextFormater::putWhites(std::ostream& status, std::ostream& output, bool wide,
482 const std::string& endSentenceTag, std::ostringstream& tagValue,
483 std::ostream& carLuTarget, char carLu, char fillChar,
484 uint64_t nb, uint64_t nbNewLines) const
485{
486 LIMA_UNUSED(status);
487 LIMA_UNUSED(wide);
488 std::string s;
489
490 s = std::string(nb/2, fillChar);
491 s.append(nbNewLines, '\n');
492 if (nb%2 == 0) s.append(nb/2, fillChar);
493 else s.append(nb/2 + 1, fillChar);
494 if ( (endSentenceTag != "")
495 && (tagValue.str().find(endSentenceTag,0) != std::string::npos)
496 && (s.length() > 0) )
497 s[0] = '.';
498 output << s;
499 nb = 0;
500 nbNewLines = 0;
501 carLuTarget << carLu;
502}
503
504} // closing namespace Misc
505} // closing Common
506} // closing namespace Lima
LIMA_UNUSED
#define LIMA_UNUSED(x)
Definition LimaCommon.h:224
LDEBUG
#define LDEBUG
Definition LimaCommon.h:157
Lima::Common::Misc::TextFormater::untaggingWithSpaces
int untaggingWithSpaces(std::ostream &status, std::istream &input, std::ostream &output, bool wide=false, const std::string &endSentenceTag="") const
This method replaces all the tags in the UTF-8 encoded input stream by spaces or underscores in the o...
Definition textFormater.cpp:55
Lima
NAUTITIA.
Definition AmosePluginsManager.h:15
LILOGINIT
#define LILOGINIT
Definition textFormater.cpp:27