RnnTokenizer.cpp

Go to the documentation of this file.

// SPDX-FileCopyrightText: 2022 CEA LIST <gael.de-chalendar@cea.fr>
//
// SPDX-License-Identifier: MIT
 
#include <QtCore/QTemporaryFile>
#include <QtCore/QRegularExpression>
#include <QDir>
 
#include "common/misc/Exceptions.h"
#include "common/Data/strwstrtools.h"
#include "common/AbstractFactoryPattern/SimpleFactory.h"
#include "common/tools/FileUtils.h"
#include "common/MediaticData/mediaticData.h"
#include "common/time/timeUtilsController.h"
 
#include "linguisticProcessing/core/LinguisticResources/LinguisticResources.h"
#include "linguisticProcessing/common/linguisticData/LimaStringText.h"
#include "linguisticProcessing/core/LinguisticAnalysisStructure/AnalysisGraph.h"
#include "linguisticProcessing/core/TextSegmentation/SegmentationData.h"
#include "linguisticProcessing/common/helpers/ConfigurationHelper.h"
#include "linguisticProcessing/common/helpers/DeepTokenizerBase.h"
#include "linguisticProcessing/common/helpers/LangCodeHelpers.h"
 
#include "RnnTokenizer.h"
 
#include "deeplima/segmentation.h"
#include "deeplima/token_type.h"
 
 
 
#define DEBUG_THIS_FILE true
 
using namespace Lima::Common::XMLConfigurationFiles;
using namespace Lima::Common::PropertyCode;
using namespace Lima::Common::MediaticData;
using namespace Lima::Common::Misc;
using namespace Lima::LinguisticProcessing::LinguisticAnalysisStructure;
 
using namespace deeplima;
 
namespace Lima
{
namespace LinguisticProcessing
{
namespace DeepLimaUnits
{
namespace RnnTokenizer
{
 
static SimpleFactory<MediaProcessUnit, RnnTokenizer> rnntokenizerFactory(RNNTOKENIZER_CLASSID); // clazy:exclude=non-pod-global-static
 
#if defined(DEBUG_LP) && defined(DEBUG_THIS_FILE)
  #define LOG_MESSAGE(stream, msg) stream << msg;
  #define LOG_MESSAGE_WITH_PROLOG(stream, msg) TOKENIZERLOGINIT; LOG_MESSAGE(stream, msg);
#else
  #define LOG_MESSAGE(stream, msg) ;
  #define LOG_MESSAGE_WITH_PROLOG(stream, msg) ;
#endif
 
CONFIGURATIONHELPER_LOGGING_INIT(TOKENIZERLOGINIT);
 
class RnnTokenizerPrivate : public DeepTokenizerBase, public ConfigurationHelper
{
  friend RnnTokenizer;
public:
  RnnTokenizerPrivate();
  virtual ~RnnTokenizerPrivate();
 
  struct TPrimitiveToken
  {
    TPrimitiveToken() { }
    TPrimitiveToken(const QString& w,
                    int pos,
                    const QString& orig=QString())
        : wordText(w), originalText(orig), start(pos)
    { }
 
    QString wordText;
    QString originalText;
    int start;
  };
 
  void init(GroupConfigurationStructure& unitConfiguration);
  void tokenize(const QString& text, std::vector<std::vector<TPrimitiveToken>>& sentences);
 
protected:
  void append_new_word(std::vector< TPrimitiveToken >& current_sentence,
                       const QString& current_token,
                       int current_token_offset) const;
 
  MediaId m_language;
  FsaStringsPool* m_stringsPool;
  LinguisticGraphVertex m_currentVx;
  QString m_data;
 
  size_t m_max_seq_len;
 
  std::map<QString, std::vector<QString>> m_trrules;
 
  // Parameters
  bool m_ignoreEOL;
 
  std::shared_ptr<segmentation::impl::SegmentationImpl> m_segm;
 
  std::function<void()> m_load_fn;
  bool m_loaded;
};
 
RnnTokenizerPrivate::RnnTokenizerPrivate() :
  ConfigurationHelper("RnnTokenizerPrivate", THIS_FILE_LOGGING_CATEGORY()),
  m_stringsPool(nullptr),
  m_currentVx(0),
  m_ignoreEOL(false),
  m_segm(nullptr),
  m_loaded(false)
{
}
 
RnnTokenizerPrivate::~RnnTokenizerPrivate()
{
}
 
RnnTokenizer::RnnTokenizer() :
  m_d(new RnnTokenizerPrivate())
{
}
 
RnnTokenizer::~RnnTokenizer()
{
  delete m_d;
}
 
void RnnTokenizer::init(
  GroupConfigurationStructure& unitConfiguration,
  Manager* manager)
{
  LOG_MESSAGE_WITH_PROLOG(LDEBUG, "RnnTokenizer::init");
 
  m_d->m_language = manager->getInitializationParameters().media;
  m_d->m_stringsPool = &MediaticData::changeable().stringsPool(m_d->m_language);
 
  m_d->init(unitConfiguration);
}
 
LimaStatusCode RnnTokenizer::process(AnalysisContent& analysis) const
{
  TimeUtils::updateCurrentTime();
  LOG_MESSAGE_WITH_PROLOG(LINFO, "start tokenizer process");
  TimeUtilsController RnnTokenizerProcessTime("RnnTokenizer");
 
 
  auto anagraph = std::make_shared<AnalysisGraph>("AnalysisGraph", m_d->m_language, true, true);
  analysis.setData("AnalysisGraph", anagraph);
  auto graph = anagraph->getGraph();
  m_d->m_currentVx = anagraph->firstVertex();
  // Get text from analysis
  auto originalText = std::dynamic_pointer_cast<LimaStringText>(analysis.getData("Text"));
  if (originalText == nullptr)
  {
      TOKENIZERLOGINIT;
      LERROR << "Can't Process RnnTokenizer: missing data 'Text'";
      return MISSING_DATA;
  }
 
  // Execute model on the text
  std::vector< std::vector< RnnTokenizerPrivate::TPrimitiveToken > > sentencesTokens;
  m_d->tokenize(*originalText, sentencesTokens);
    LOG_MESSAGE(LDEBUG, "      Number of token '" << sentencesTokens.size() << "'");
 
  // Insert the tokens in the graph and create sentence limits
  auto sb = std::make_shared<SegmentationData>("AnalysisGraph");
  analysis.setData(m_d->m_data.toStdString(), sb);
 
  // Remove the default 0 -> 1 edge before adding tokens
  remove_edge(anagraph->firstVertex(),
              anagraph->lastVertex(),
              *graph);
  LinguisticGraphVertex beginSentence = 0;
 
  // Insert the tokens in the graph and create sentence limits
  for (const auto& sentence: sentencesTokens)
  {
    if (sentence.size() < 1)
    {
      continue;
    }
 
    auto endSentence = std::numeric_limits< LinguisticGraphVertex >::max();
    for (const auto& token: sentence)
    {
      const auto& str = token.wordText;
 
      LOG_MESSAGE(LDEBUG, "      Adding token '" << str << "'");
 
      auto form = (*m_d->m_stringsPool)[str];
      Token *tToken = new Token(form, str, token.start+1, token.wordText.size());
      if (tToken == nullptr)
      {
        TOKENIZERLOGINIT;
        LERROR << "RnnFlowTokenizer::process: Can't allocate memory with \"new Token(...)\"";
        throw MemoryErrorException();
      }
 
      if (token.originalText.size() > 0)
      {
        // tranduced token
        // save original word as orthographic alternative
        auto orig = (*m_d->m_stringsPool)[token.originalText];
        tToken->addOrthographicAlternatives(orig);
      }
 
      m_d->computeDefaultStatus(*tToken);
 
      LOG_MESSAGE(LDEBUG, "      status is " << tToken->status().toString());
 
      // Adds on the path
      auto newVx = add_vertex(*graph);
      endSentence = newVx;
      put(vertex_token, *graph, newVx, tToken);
      put(vertex_data, *graph, newVx, new MorphoSyntacticData());
      add_edge(m_d->m_currentVx, newVx, *graph);
      m_d->m_currentVx = newVx;
    }
 
    LOG_MESSAGE(LDEBUG, "adding sentence" << beginSentence << endSentence);
 
    sb->add(Segment("sentence", beginSentence, endSentence, anagraph.get()));
    beginSentence = endSentence;
  }
 
  add_edge(m_d->m_currentVx, anagraph->lastVertex(), *graph);
 
  TimeUtils::logElapsedTime("RnnTokenizer");
  return SUCCESS_ID;
}
 
void RnnTokenizerPrivate::init(GroupConfigurationStructure& unitConfiguration)
{
  m_data = QString(getStringParameter(unitConfiguration, "data", 0, "SentenceBoundaries").c_str());
  auto model_prefix = QString::fromStdString(
    getStringParameter(unitConfiguration, "model_prefix",
                       ConfigurationHelper::REQUIRED | ConfigurationHelper::NOT_EMPTY));
 
  LOG_MESSAGE_WITH_PROLOG(LDEBUG, "RnnTokenizerPrivate::init" << model_prefix);
 
  auto lang_str = QString::fromStdString(MediaticData::single().media(m_language));
  auto resources_path = QString::fromStdString(MediaticData::single().getResourcesPath());
  auto model_name = model_prefix;
  std::string udlang;
  MediaticData::single().getOptionValue("udlang", udlang);
  LOG_MESSAGE(LDEBUG, "RnnTokenizerPrivate::init lang_str=" << lang_str << ", udlang=" << udlang);
 
  if (!fix_lang_codes(lang_str, udlang))
  {
    LIMA_EXCEPTION_SELECT_LOGINIT(TOKENIZERLOGINIT,
      "RnnTokenizerPrivate::init: Can't parse language id " << udlang.c_str(),
      Lima::InvalidConfiguration);
  }
 
  model_name.replace(QString("$udlang"), QString::fromStdString(udlang));
 
  auto model_file_name = findFileInPaths(resources_path,
                                         QString::fromUtf8("/RnnTokenizer/%1/%2.pt")
                                           .arg(lang_str).arg(model_name));
  if (model_file_name.isEmpty())
  {
    throw InvalidConfiguration("RnnTokenizerPrivate::init: tokenizer model file not found.");
  }
 
  m_load_fn = [this, model_file_name]()
  {
    // if (m_loaded)
    // {
    //   return;
    // }
 
    m_segm->load(model_file_name.toStdString());
    m_segm->init(1, 16*1024); // threads, buffer size per thread
 
    m_loaded = true;
  };
 
  // if (!isInitLazy())
  // {
  //   m_load_fn();
  // }
}
 
void RnnTokenizerPrivate::append_new_word(std::vector< TPrimitiveToken >& current_sentence,
                                          const QString& current_token,
                                          int current_token_offset) const
{
  auto ctoken_lower = current_token.toLower();
 
  auto i = m_trrules.find(ctoken_lower);
  if (i == m_trrules.end())
  {
    current_sentence.push_back(TPrimitiveToken(current_token, current_token_offset));
  }
  else
  {
    size_t n = 0;
    for (const auto& w : i->second)
    {
      if (n == 0)
      {
        current_sentence.push_back(TPrimitiveToken(w, current_token_offset, current_token));
      }
      else
      {
        current_sentence.push_back(TPrimitiveToken(w, current_token_offset));
      }
    }
  }
}
 
void RnnTokenizerPrivate::tokenize(const QString& text, std::vector<std::vector<TPrimitiveToken>>& sentences)
{
  m_segm = std::make_shared<segmentation::impl::SegmentationImpl>();
 
  m_load_fn();
 
  LOG_MESSAGE_WITH_PROLOG(LDEBUG, "RnnTokenizerPrivate::tokenize" << text.left(100));
  TimeUtils::updateCurrentTime();
  sentences.clear();
  sentences.reserve(text.size() / 15);
 
  std::vector< TPrimitiveToken > current_sentence;
  int current_token_offset = 0;
 
  auto text_utf8 = text.toStdString();
 
  m_segm->register_handler([this, &sentences, &current_sentence, &current_token_offset]
                          (const std::vector<segmentation::token_pos>& tokens,
                           uint32_t len)
  {
    for (size_t i = 0; i < len; i++)
    {
      const auto& tok = tokens[i];
      if (0 == tok.m_len)
      {
        continue;
      }
      append_new_word(current_sentence, QString::fromUtf8(tok.m_pch, tok.m_len), current_token_offset);
      current_token_offset += (tok.m_offset + tok.m_len);
      if (tok.m_flags & token_flags_t::sentence_brk)
      {
        sentences.push_back(current_sentence);
        current_sentence.clear();
      }
    }
  });
 
  size_t bytes_consumed = 0;
  m_segm->parse_from_stream([&text_utf8, &bytes_consumed]
                         (uint8_t* buffer,
                         int32_t& read,
                         size_t max)
  {
    read = (text_utf8.size() - bytes_consumed) > max ? max : (text_utf8.size() - bytes_consumed);
    memcpy(buffer, text_utf8.c_str() + bytes_consumed, read);
    bytes_consumed += read;
    return (text_utf8.size() - bytes_consumed) > max;
  });
 
  LOG_MESSAGE(LINFO, "End of Tokenizer");
  TimeUtils::logElapsedTime("RnnTokenizerPrivate");
}
 
} // namespace RnnTokenizer
} // namespace DeepLimaUnits
} // namespace LinguisticProcessing
} // namespace Lima