RnnTokensAnalyzer.cpp

Go to the documentation of this file.

// SPDX-FileCopyrightText: 2022 CEA LIST <gael.de-chalendar@cea.fr>
//
// SPDX-License-Identifier: MIT
 
#include <QtCore/QTemporaryFile>
#include <QtCore/QRegularExpression>
#include <QDir>
 
 
#include "common/AbstractFactoryPattern/SimpleFactory.h"
#include "common/tools/FileUtils.h"
#include "common/MediaticData/mediaticData.h"
#include "common/time/timeUtilsController.h"
 
#include "linguisticProcessing/common/linguisticData/LimaStringText.h"
#include "linguisticProcessing/core/LinguisticAnalysisStructure/AnalysisGraph.h"
#include "linguisticProcessing/common/helpers/ConfigurationHelper.h"
#include "linguisticProcessing/common/helpers/LangCodeHelpers.h"
#include "linguisticProcessing/common/annotationGraph/AnnotationData.h"
#include "linguisticProcessing/core/LinguisticAnalysisStructure/LinguisticGraph.h"
#include "linguisticProcessing/core/LinguisticAnalysisStructure/MorphoSyntacticData.h"
#include "linguisticProcessing/core/LinguisticAnalysisStructure/MorphoSyntacticDataUtils.h"
 
#include "RnnTokensAnalyzer.h"
#include "deeplima/token_sequence_analyzer.h"
#include "deeplima/token_type.h"
#include "deeplima/dumper_conllu.h"
#include "helpers/path_resolver.h"
#include "linguisticProcessing/core/DeepLimaUnits/TokenIteratorData.h"
#include "linguisticProcessing/core/TextSegmentation/SegmentationData.h"
 
#include <set>
 
#define DEBUG_THIS_FILE true
 
using namespace Lima::Common::XMLConfigurationFiles;
using namespace Lima::Common::PropertyCode;
using namespace Lima::Common::MediaticData;
using namespace Lima::Common::Misc;
using namespace Lima::LinguisticProcessing::LinguisticAnalysisStructure;
using namespace Lima::Common::AnnotationGraphs;
using namespace std;
using namespace deeplima;
 
 
namespace Lima::LinguisticProcessing::DeepLimaUnits::RnnTokensAnalyzer
{
#if defined(DEBUG_LP) && defined(DEBUG_THIS_FILE)
#define LOG_MESSAGE(stream, msg) stream << msg;
#define LOG_MESSAGE_WITH_PROLOG(stream, msg) LEMMALOGINIT; LOG_MESSAGE(stream, msg);
#else
    #define LOG_MESSAGE(stream, msg) ;
    #define LOG_MESSAGE_WITH_PROLOG(stream, msg) ;
#endif
 
static SimpleFactory<MediaProcessUnit, RnnTokensAnalyzer> RnnTokensAnalyzerFactory(RNNTOKENSANALYZER_CLASSID); // clazy:exclude=non-pod-global-static
 
CONFIGURATIONHELPER_LOGGING_INIT(LEMMALOGINIT);
 
class RnnTokensAnalyzerPrivate: public ConfigurationHelper
{
public:
    RnnTokensAnalyzerPrivate();
    ~RnnTokensAnalyzerPrivate() ;
    void init(GroupConfigurationStructure& unitConfiguration);
    void analyzer(vector<segmentation::token_pos>& buffer);
    void insertTokenInfo(TokenSequenceAnalyzer<>::TokenIterator &ti);
    dumper::AnalysisToConllU<TokenSequenceAnalyzer<>::TokenIterator> m_dumper;
    void reset();
 
    Lima::AnalysisContent* m_analysis;
    MediaId m_language;
    FsaStringsPool* m_stringsPool;
    QString m_data;
    std::shared_ptr< TokenSequenceAnalyzer<> > m_tokensAnalyzer;
    std::function<void()> m_load_fn;
    StringIndex m_stridx;
    PathResolver m_pResolver;
    std::vector< std::map<std::string, std::string> > m_tags;
    std::vector<QString> m_lemmas;
    const Common::PropertyCode::PropertyAccessor* m_microAccessor;
    bool m_loaded;
    // TokenSequenceAnalyzer<>::OutputMatrix* m_matrix;
};
 
RnnTokensAnalyzerPrivate::RnnTokensAnalyzerPrivate(): ConfigurationHelper(
    "RnnTokensAnalyzerPrivate", THIS_FILE_LOGGING_CATEGORY()),
    m_stringsPool(nullptr), m_stridx(),
    m_pResolver(MediaticData::single().getResourcesPath()),
    m_loaded(false)
{
}
 
RnnTokensAnalyzerPrivate::~RnnTokensAnalyzerPrivate() = default;
 
 
RnnTokensAnalyzer::RnnTokensAnalyzer(): m_d(new RnnTokensAnalyzerPrivate())
{
}
 
RnnTokensAnalyzer::~RnnTokensAnalyzer()
{
    delete m_d;
}
 
void RnnTokensAnalyzer::init(GroupConfigurationStructure &unitConfiguration, Manager *manager)
{
    LOG_MESSAGE_WITH_PROLOG(LDEBUG, "RnnTokensAnalyzer::init");
 
    m_d->m_language = manager->getInitializationParameters().media;
    m_d->m_stringsPool = &MediaticData::changeable().stringsPool(m_d->m_language);
 
    m_d->init(unitConfiguration);
}
 
Lima::LimaStatusCode RnnTokensAnalyzer::process(Lima::AnalysisContent &analysis) const
{
    TimeUtils::updateCurrentTime();
    TimeUtilsController RnnTokensAnalyzerProcessTime("RnnTokensAnalyzer");
    LOG_MESSAGE_WITH_PROLOG(LDEBUG, "RnnTokensAnalyzer::process");
    m_d->m_analysis = &analysis;
    auto anagraph = std::dynamic_pointer_cast<AnalysisGraph>(analysis.getData("AnalysisGraph"));
    if (anagraph == nullptr)
    {
        LEMMALOGINIT;
        LERROR << "Can't Process RnnTokensAnalyzer: missing data 'AnalysisGraph'";
        return MISSING_DATA;
    }
    auto srcgraph = anagraph->getGraph();
    auto endVx = anagraph->lastVertex();
    auto posgraph = new LinguisticAnalysisStructure::AnalysisGraph("PosGraph",
                                                                    m_d->m_language,
                                                                    false,
                                                                    true);
    analysis.setData("PosGraph", posgraph);
    const auto& propertyCodeManager = dynamic_cast<const LanguageData&>(
            MediaticData::single().mediaData(m_d->m_language)).getPropertyCodeManager();
    const auto& microManager = propertyCodeManager.getPropertyManager("MICRO");
    auto annotationData = std::dynamic_pointer_cast< AnnotationData >(analysis.getData("AnnotationData"));
    if (annotationData == nullptr)
    {
        annotationData = std::make_shared<AnnotationData>();
        if (std::dynamic_pointer_cast<AnalysisGraph>(analysis.getData("AnalysisGraph")) != nullptr)
        {
            std::dynamic_pointer_cast<AnalysisGraph>(analysis.getData("AnalysisGraph"))->populateAnnotationGraph(
                    annotationData.get(), "AnalysisGraph");
        }
        analysis.setData("AnnotationData", annotationData);
    }
    if (num_vertices(*srcgraph)<=2)
    {
        return SUCCESS_ID;
    }
 
    auto vTokens = get(vertex_token, *srcgraph);
    auto currentVx = anagraph->firstVertex();
    auto resultgraph = posgraph->getGraph();
    remove_edge(posgraph->firstVertex(), posgraph->lastVertex(), *resultgraph);
 
    // Sentence boundaries are stored in SegmentationData (not in token statuses),
    // so collect the last real token of each sentence here. We mark those with the
    // deeplima sentence_brk flag so downstream units fed by this token stream
    // (e.g. the dependency parser) split the text per sentence instead of parsing
    // it as one long sequence. Segment::getLastVertex() is the vertex *after* the
    // last token, so the sentence-final token is its in-neighbour.
    std::set<LinguisticGraphVertex> sentenceFinalVertices;
    {
        auto sd = std::dynamic_pointer_cast<SegmentationData>(
            analysis.getData("SentenceBoundaries"));
        if (sd != nullptr)
        {
            for (const auto& segment : sd->getSegments())
            {
                const auto endV = segment.getLastVertex();
                if (vTokens[endV] != nullptr)
                {
                    // getLastVertex() is itself the sentence's last token.
                    sentenceFinalVertices.insert(endV);
                }
                else
                {
                    // getLastVertex() is the boundary after the last token: mark
                    // its in-neighbour token(s).
                    LinguisticGraphInEdgeIt ie, ie_end;
                    for (boost::tie(ie, ie_end) = boost::in_edges(endV, *srcgraph);
                         ie != ie_end; ++ie)
                    {
                        const auto s = boost::source(*ie, *srcgraph);
                        if (vTokens[s] != nullptr)
                        {
                            sentenceFinalVertices.insert(s);
                        }
                    }
                }
            }
        }
    }
 
    std::vector<segmentation::token_pos> buffer;
    std::vector< LinguisticGraphVertex > anaVertices;
    std::vector<std::string> v;
 
    while(currentVx != endVx)
    {
        if (currentVx != 0 && vTokens[currentVx] != nullptr)
        {
            const auto& src = vTokens[currentVx];
            v.push_back(src->stringForm().toStdString());
            buffer.emplace_back();
            anaVertices.push_back(currentVx);
        }
        LinguisticGraphOutEdgeIt it, it_end;
        boost::tie(it, it_end) = boost::out_edges(currentVx, *srcgraph);
        if (it != it_end)
        {
            currentVx = boost::target(*it, *srcgraph);
        }
        else
        {
            currentVx = endVx;
        }
    }
    for(unsigned long k = 0; k < anaVertices.size(); k++)
    {
        currentVx = anaVertices[k];
        if (currentVx != 0 && vTokens[currentVx] != nullptr)
        {
            const auto& src = vTokens[currentVx];
            auto& token = buffer[k];
            token.m_offset = src->position();
            // deeplima's token_pos::m_len is a BYTE length and m_pch points to the
            // UTF-8 bytes of v[k]. src->length() is the LIMA character count, which
            // truncates multi-byte UTF-8 forms (e.g. "à" → 1 instead of 2 bytes),
            // leaving the downstream form an incomplete byte sequence that decodes
            // to an empty string (and crashed the neural lemmatizer). Use the UTF-8
            // byte length so m_len matches m_pch.
            token.m_len = v[k].size();
            token.m_pch = v[k].c_str();
            // Mark the last token of each sentence (from SegmentationData) so the
            // stream carries real sentence breaks. The previous code AND-ed the
            // sequential StatusType enum against a deeplima bit flag, which never
            // produced a valid sentence_brk.
            token.m_flags = (sentenceFinalVertices.count(currentVx) > 0)
                                ? token_flags_t::sentence_brk
                                : token_flags_t::none;
        }
    }
    m_d->analyzer(buffer);
    std::vector<LinguisticGraphVertex>::size_type anaVerticesIndex = 0;
    auto previousPosVertex = posgraph->firstVertex();
    /*
     * Here we add the part of speech data to the tokens
     * Adding link beetween the node in the analysis graph and the pos graph.
     */
    while (anaVerticesIndex < anaVertices.size())
    {
        auto anaVertex = anaVertices[anaVerticesIndex];
        auto newVx = boost::add_vertex(*resultgraph);
        auto agv =  annotationData->createAnnotationVertex();
        annotationData->addMatching("PosGraph", newVx, "annot", agv);
        annotationData->addMatching("AnalysisGraph", anaVertex, "PosGraph", newVx);
        annotationData->annotate(agv, QString::fromUtf8("PosGraph"), newVx);
 
        auto srcToken = get(vertex_token,*srcgraph,anaVertex);
 
        auto posData = new MorphoSyntacticData();
 
        LinguisticElement lElement = LinguisticElement();
        for(const auto& name: m_d->m_tokensAnalyzer->get_class_names())
        {
            auto propertyManager = microManager;
            if(name == "upos")
            {
                propertyManager = propertyCodeManager.getPropertyManager("MICRO");
            }
            else if(name=="ExtPos" || name=="Style")
            {
                continue;
            }
            else if(name=="xpos")
            {
                propertyManager = propertyCodeManager.getPropertyManager("MACRO");
            }
            else
            {
                propertyManager = propertyCodeManager.getPropertyManager(name);
            }
            auto value = propertyManager.getPropertyValue(m_d->m_tags[anaVerticesIndex][name]);
            if(value.toBool())
            {
                LOG_MESSAGE(LDEBUG, "value: " << value);
                propertyManager.getPropertyAccessor().writeValue(value, lElement.properties);
            }
        }
        auto sp = &MediaticData::changeable().stringsPool(m_d->m_language);
        lElement.lemma = (*sp)[m_d->m_lemmas[anaVerticesIndex]];
        posData->push_back(lElement);
 
        put(vertex_data, *resultgraph, newVx, posData);
        put(vertex_token, *resultgraph, newVx, srcToken);
 
        boost::add_edge(previousPosVertex, newVx, *resultgraph);
 
        previousPosVertex = newVx;
        anaVerticesIndex++;
    }
    boost::add_edge(previousPosVertex, posgraph->lastVertex(), *resultgraph);
    LOG_MESSAGE(LDEBUG, "RnnPosLemmatizer postagging done.");
    TimeUtils::logElapsedTime("RnnTokensAnalyzer");
    return SUCCESS_ID;
}
 
void RnnTokensAnalyzerPrivate::init(GroupConfigurationStructure& unitConfiguration)
{
    m_data = QString(getStringParameter(unitConfiguration, "data", 0, "SentenceBoundaries").c_str());
    QString tagger_model_prefix = getStringParameter(unitConfiguration, "tagger_model_prefix", ConfigurationHelper::REQUIRED | ConfigurationHelper::NOT_EMPTY).c_str();
    QString lemmatizer_model_prefix = getStringParameter(unitConfiguration, "lemmatizer_model_prefix", ConfigurationHelper::REQUIRED | ConfigurationHelper::NOT_EMPTY).c_str();
    LOG_MESSAGE_WITH_PROLOG(LDEBUG, "RnnTokensAnalyzerPrivate::init tagger model: " << tagger_model_prefix << " lemmatizer model: " << lemmatizer_model_prefix);
 
    QString lang_str = MediaticData::single().media(m_language).c_str();
    QString resources_path = MediaticData::single().getResourcesPath().c_str();
    QString tagger_model_name = tagger_model_prefix;
    QString lemmatizer_model_name = lemmatizer_model_prefix;
    string udlang;
    MediaticData::single().getOptionValue("udlang", udlang);
 
    if (!fix_lang_codes(lang_str, udlang))
    {
        LIMA_EXCEPTION_SELECT_LOGINIT(LEMMALOGINIT,
                                        "RnnTokensAnalyzerPrivate::init: Can't parse language id " << udlang.c_str(),
                                        Lima::InvalidConfiguration);
    }
 
    tagger_model_name.replace(QString("$udlang"), QString(udlang.c_str()));
    lemmatizer_model_name.replace(QString("$udlang"), QString(udlang.c_str()));
 
    auto tagger_model_file_name = findFileInPaths(resources_path,
                                                    QString::fromUtf8("/RnnTagger/%1/%2.pt")
                                                    .arg(lang_str, tagger_model_name));
    auto lemmatizer_model_file_name = findFileInPaths(resources_path,
                                                    QString::fromUtf8("/RnnLemmatizer/%1/%2.pt")
                                                            .arg(lang_str, lemmatizer_model_name));
    auto lemmatizer_dictionary_file_name = findFileInPaths(resources_path,
                                                           QString::fromUtf8("/RnnLemmatizer/%1/%2.dic")
                                                            .arg(lang_str, lemmatizer_model_name));
    if (tagger_model_file_name.isEmpty())
    {
        throw InvalidConfiguration("RnnTokensAnalyzerPrivate::init: tagger model file not found.");
    }
    if (lemmatizer_model_file_name.isEmpty())
    {
        //throw InvalidConfiguration("RnnTokensAnalyzerPrivate::init: lemmatizer model file not found.");
        lemmatizer_model_file_name = "";
    }
 
    m_load_fn = [this, tagger_model_file_name, lemmatizer_model_file_name, lemmatizer_dictionary_file_name]()
    {
        if (m_loaded)
        {
            return;
        }
        // TODO give the correct parameters for fixed_ini, lower_ini and lower_lemm
        m_tokensAnalyzer = std::make_shared< TokenSequenceAnalyzer<> >(tagger_model_file_name.toStdString(),
                                                                       lemmatizer_model_file_name.toStdString(),
                                                                       lemmatizer_dictionary_file_name.toStdString(),
                                                                       "", "", "",
                                                                       m_pResolver, 1024, 8);
        m_loaded = true;
    };
 
    if (!isInitLazy())
    {
        m_load_fn();
    }
    for (size_t i = 0; i < m_tokensAnalyzer->get_classes().size(); ++i)
    {
        m_dumper.set_classes(i, m_tokensAnalyzer->get_class_names()[i], m_tokensAnalyzer->get_classes()[i]);
    }
 
}
 
void RnnTokensAnalyzerPrivate::reset()
{
    m_tags.clear();
    m_lemmas.clear();
}
 
void RnnTokensAnalyzerPrivate::analyzer(std::vector<segmentation::token_pos> &buffer)
{
    reset();
    m_tokensAnalyzer->register_handler([this](std::shared_ptr< StringIndex > stridx,
                                                const token_buffer_t<>& tokens,
                                                const std::vector<StringIndex::idx_t>& lemmata,
                                                std::shared_ptr< StdMatrix<uint8_t> > classes,
                                                size_t begin,
                                                size_t end)
    {
        auto ti = std::make_shared<TokenSequenceAnalyzer<>::TokenIterator>(*stridx, tokens, lemmata, classes, begin, end);
        auto tiData = std::make_shared<TokenIteratorData>();
        tiData->setTokenIterator(ti);
        // auto si = std::make_shared<deeplima::StringIndex>(stridx);
        tiData->setStringIndex(stridx);
        m_analysis->setData("TokenIterator", tiData);
        // m_matrix = new TokenSequenceAnalyzer<>::OutputMatrix(classes);
        insertTokenInfo(*ti);
    });
    LOG_MESSAGE_WITH_PROLOG(LDEBUG, buffer[0].m_pch);
    (*m_tokensAnalyzer)(buffer, buffer.size());
    m_tokensAnalyzer->finalize();
}
 
void RnnTokensAnalyzerPrivate::insertTokenInfo(TokenSequenceAnalyzer<>::TokenIterator &ti)
{
    auto classes = m_dumper.getMClasses();
    auto class_names = m_tokensAnalyzer->get_class_names();
    LOG_MESSAGE_WITH_PROLOG(LDEBUG, "classes: " << class_names);
    while(!ti.end())
    {
        auto tag = std::map<std::string, std::string>();
        LOG_MESSAGE(LDEBUG, ti.lemma());
        for(uint cat = 0; cat < class_names.size(); cat++)
        {
            tag.insert({class_names[cat], classes[cat][ti.token_class(cat)]});
        }
        m_tags.push_back(tag);
        m_lemmas.emplace_back(ti.lemma());
        ti.next();
    }
    ti.reset();
}
 
 
}