Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "trparser.h"
#include "simtexth.h"
#include "translator.h"
#include <QtCore/QCoreApplication>
#include <QtCore/QDebug>
#include <QtCore/QList>
#include <QtCore/QMap>
#include <QtCore/QStringList>
QT_BEGIN_NAMESPACE
using namespace Qt::Literals::StringLiterals;
/*
Augments a Translator with trivially derived translations.
For example, if "Enabled:" is consistendly translated as "Eingeschaltet:" no
matter the context or the comment, "Eingeschaltet:" is added as the
translation of any untranslated "Enabled:" text and is marked Unfinished.
Returns the number of additional messages that this heuristic translated.
*/
int applySameTextHeuristic(Translator &tor)
{
QMap<QString, QStringList> translated;
QMap<QString, bool> avoid; // Want a QTreeSet, in fact
QList<bool> untranslated(tor.messageCount());
int inserted = 0;
for (int i = 0; i < tor.messageCount(); ++i) {
const TranslatorMessage &msg = tor.message(i);
if (!msg.isTranslated()) {
if (msg.type() == TranslatorMessage::Unfinished)
untranslated[i] = true;
} else {
const QString &key = msg.sourceText();
const auto t = translated.constFind(key);
if (t != translated.constEnd()) {
/*
The same source text is translated at least two
different ways. Do nothing then.
*/
if (*t != msg.translations()) {
translated.remove(key);
avoid.insert(key, true);
}
} else if (!avoid.contains(key)) {
translated.insert(key, msg.translations());
}
}
}
for (int i = 0; i < tor.messageCount(); ++i) {
if (untranslated[i]) {
TranslatorMessage &msg = tor.message(i);
const auto t = translated.constFind(msg.sourceText());
if (t != translated.constEnd()) {
msg.setTranslations(*t);
++inserted;
}
}
}
return inserted;
}
/*
Merges two Translator objects. The first one
is a set of source texts and translations for a previous version of
the internationalized program; the second one is a set of fresh
source texts newly extracted from the source code, without any
translation yet.
*/
Translator merge(
const Translator &tor, const Translator &virginTor, const QList<Translator> &aliens,
UpdateOptions options, QString &err)
{
int known = 0;
int neww = 0;
int obsoleted = 0;
int similarTextHeuristicCount = 0;
Translator outTor;
outTor.setLanguageCode(tor.languageCode());
outTor.setSourceLanguageCode(tor.sourceLanguageCode());
outTor.setLocationsType(tor.locationsType());
/*
The types of all the messages from the vernacular translator
are updated according to the virgin translator.
*/
for (TranslatorMessage m : tor.messages()) {
TranslatorMessage::Type newType = TranslatorMessage::Finished;
TranslatorMessage::ExtraData extras;
const TranslatorMessage *mv;
int mvi = virginTor.find(m);
if (mvi < 0) {
if (!(options & HeuristicSimilarText)) {
makeObsolete:
switch (m.type()) {
case TranslatorMessage::Finished:
newType = TranslatorMessage::Vanished;
obsoleted++;
break;
case TranslatorMessage::Unfinished:
newType = TranslatorMessage::Obsolete;
obsoleted++;
break;
default:
newType = m.type();
break;
}
m.clearReferences();
} else {
mvi = virginTor.find(m.context(), m.comment(), m.allReferences());
if (mvi < 0) {
// did not find it in the virgin, mark it as obsolete
goto makeObsolete;
}
mv = &virginTor.constMessage(mvi);
// Do not just accept it if its on the same line number,
// but different source text.
// Also check if the texts are more or less similar before
// we consider them to represent the same message...
if (getSimilarityScore(m.sourceText(), mv->sourceText()) < textSimilarityThreshold) {
// The virgin and vernacular sourceTexts are so different that we could not find it
goto makeObsolete;
}
// It is just slightly modified, assume that it is the same string
extras = mv->extras();
// Mark it as unfinished. (Since the source text
// was changed it might require re-translating...)
newType = TranslatorMessage::Unfinished;
++similarTextHeuristicCount;
neww++;
goto outdateSource;
}
} else {
mv = &virginTor.message(mvi);
extras = mv->extras();
if (!mv->id().isEmpty()
&& (mv->context() != m.context()
|| mv->sourceText() != m.sourceText()
|| mv->comment() != m.comment())) {
known++;
newType = TranslatorMessage::Unfinished;
m.setContext(mv->context());
m.setComment(mv->comment());
if (mv->sourceText() != m.sourceText()) {
outdateSource:
m.setOldSourceText(m.sourceText());
m.setSourceText(mv->sourceText());
const QString &oldpluralsource = m.extra("po-msgid_plural"_L1);
if (!oldpluralsource.isEmpty())
extras.insert("po-old_msgid_plural"_L1, oldpluralsource);
}
} else {
switch (m.type()) {
case TranslatorMessage::Finished:
default:
if (m.isPlural() == mv->isPlural()) {
newType = TranslatorMessage::Finished;
} else {
newType = TranslatorMessage::Unfinished;
}
known++;
break;
case TranslatorMessage::Unfinished:
newType = TranslatorMessage::Unfinished;
known++;
break;
case TranslatorMessage::Vanished:
newType = TranslatorMessage::Finished;
neww++;
break;
case TranslatorMessage::Obsolete:
newType = TranslatorMessage::Unfinished;
neww++;
break;
}
}
// Always get the filename and linenumber info from the
// virgin Translator, in case it has changed location.
// This should also enable us to read a file that does not
// have the <location> element.
// why not use operator=()? Because it overwrites e.g. userData.
m.setReferences(mv->allReferences());
m.setPlural(mv->isPlural());
m.setExtras(extras);
m.setExtraComment(mv->extraComment());
m.setId(mv->id());
m.setLabel(mv->label());
}
m.setType(newType);
outTor.append(m);
}
/*
Messages found only in the virgin translator are added to the
vernacular translator.
*/
for (const TranslatorMessage &mv : virginTor.messages()) {
if (tor.find(mv) >= 0)
continue;
if (options & HeuristicSimilarText) {
int mi = tor.find(mv.context(), mv.comment(), mv.allReferences());
if (mi >= 0) {
// The similar message found in tor (ts file) must NOT correspond exactly
// to an other message is virginTor
if (virginTor.find(tor.constMessage(mi)) < 0) {
if (getSimilarityScore(tor.constMessage(mi).sourceText(), mv.sourceText())
>= textSimilarityThreshold)
continue;
}
}
}
outTor.appendSorted(mv);
++neww;
}
/*
"Alien" translators can be used to augment the vernacular translator.
*/
for (const Translator &alf : aliens) {
for (TranslatorMessage mv : alf.messages()) {
if (mv.sourceText().isEmpty() || !mv.isTranslated())
continue;
int mvi = outTor.find(mv);
if (mvi >= 0) {
TranslatorMessage &tm = outTor.message(mvi);
if (tm.type() != TranslatorMessage::Finished && !tm.isTranslated()) {
tm.setTranslations(mv.translations());
--neww;
++known;
}
} else {
/*
* Don't do simtex search, as the locations are likely to be
* completely off anyway, so we'd find nothing.
*/
/*
* Add the unmatched messages as obsoletes, so the Linguist GUI
* will offer them as possible translations.
*/
mv.clearReferences();
mv.setType(mv.type() == TranslatorMessage::Finished
? TranslatorMessage::Vanished : TranslatorMessage::Obsolete);
outTor.appendSorted(mv);
++known;
++obsoleted;
}
}
}
/*
The same-text heuristic handles cases where a message has an
obsolete counterpart with a different context or comment.
*/
int sameTextHeuristicCount = (options & HeuristicSameText) ? applySameTextHeuristic(outTor) : 0;
if (options & Verbose) {
int totalFound = neww + known;
err += QStringLiteral(" Found %1 source text(s) (%2 new and %3 already existing)\n")
.arg(totalFound).arg(neww).arg(known);
if (obsoleted) {
if (options & NoObsolete) {
err += QStringLiteral(" Removed %1 obsolete entries\n").arg(obsoleted);
} else {
err += QStringLiteral(" Kept %1 obsolete entries\n").arg(obsoleted);
}
}
if (sameTextHeuristicCount)
err += QStringLiteral(" Same-text heuristic provided %1 translation(s)\n")
.arg(sameTextHeuristicCount);
if (similarTextHeuristicCount)
err += QStringLiteral(" Similar-text heuristic provided %1 translation(s)\n")
.arg(similarTextHeuristicCount);
}
return outTor;
}
QT_END_NAMESPACE