Source code
Revision control
Copy as Markdown
// Copyright (C) 2025 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "filetransformer.h"
#include "fileverifier.h"
#include "utils.h"
#include "recorddirectory.h"
#include <translator.h>
#include <trlib/metastrings.h>
#include <trlib/trparser.h>
#include <QFile>
#include <QFileInfo>
#include <QTextStream>
#include <QSet>
#include <QRegularExpression>
using namespace Qt::StringLiterals;
using namespace Utils;
namespace {
// UI elements for XML processing
constexpr QLatin1String uiElement = "<ui "_L1;
constexpr QLatin1String stringElement = "<string"_L1;
constexpr QLatin1String stringListElement = "<stringlist"_L1;
QString escapeForCppStringLiteral(const QString &str)
{
QString result = str;
result.replace("\\", "\\\\");
result.replace("\"", "\\\"");
return result;
}
QString textMetaString(const QString &indentation, const QString &sourceText)
{
const QStringList lines = sourceText.split('\n');
QString metaString;
for (int i = 0; i < lines.size(); i++) {
metaString += indentation + "//" + MetaStrings::sourceTextAnotation + " \""
+ escapeForCppStringLiteral(lines[i]);
if (i < lines.size() - 1)
metaString += "\\n\"\n";
else
metaString += "\"\n";
}
return metaString;
}
QString labelMetaString(const QString &indentation, const QString &label)
{
return indentation + "//" + MetaStrings::labelAnotation + ' ' + label + '\n';
}
QString idMetaString(const QString &indentation, const QString &id)
{
return indentation + "//" + MetaStrings::extraAnotation + " meta-id " + id + '\n';
}
QRegularExpression idMetaStringRegex(const QString &id)
{
const QString e = QRegularExpression::escape(id);
const QString pat = QStringLiteral(R"(^[ \t]*//~ meta-id\s*%1[ \t]*(?:\r?\n|$))").arg(e);
return QRegularExpression(pat, QRegularExpression::MultilineOption);
}
QString idBasedFunc(int trFunc, const QString &id, const QString &pluralArg)
{
switch (trFunc) {
case TrFunctionAliasManager::Function_trUtf8:
case TrFunctionAliasManager::Function_tr:
case TrFunctionAliasManager::Function_translate:
return "qtTrId(\"" + id + "\"" + pluralArg + ")";
case TrFunctionAliasManager::Function_qsTr:
case TrFunctionAliasManager::Function_qsTranslate:
return "qsTrId(\"" + id + "\"" + pluralArg + ")";
case TrFunctionAliasManager::Function_QT_TR_NOOP:
case TrFunctionAliasManager::Function_QT_TR_NOOP_UTF8:
case TrFunctionAliasManager::Function_QT_TRANSLATE_NOOP:
case TrFunctionAliasManager::Function_QT_TRANSLATE_NOOP_UTF8:
return "QT_TRID_NOOP(\"" + id + "\")";
case TrFunctionAliasManager::Function_QT_TR_N_NOOP:
case TrFunctionAliasManager::Function_QT_TRANSLATE_N_NOOP:
return "QT_TRID_N_NOOP(\"" + id + "\")";
default:
return {};
}
}
int getTrFunction(const QString &expr)
{
int i = 0;
int depth = 0;
QString trFunc;
while (i < expr.size()) {
if (QChar c = expr[i++]; (c.isLetterOrNumber() || c == '_'_L1) && depth == 0)
trFunc += c;
else {
while (c.isSpace() && i < expr.size())
c = expr[i++];
if (c == '(') {
if (depth == 0)
if (int trFuncId = trFunctionAliasManager.trFunctionByName(trFunc);
trFuncId >= 0)
return trFuncId;
depth++;
} else if (c == ')'_L1)
depth--;
else if (c == "\""_L1 || c == "'"_L1) {
const QChar quotation = c;
while (i < expr.size()) {
c = expr[i++];
if (c == '\\'_L1)
i++;
else if (c == quotation)
break;
}
}
trFunc.clear();
}
}
return -1;
}
QString getPluralArg(const QString &fn, bool plural)
{
if (!plural)
return {};
int paren = 1;
int pos = fn.size() - 2;
while (paren > 0 && pos >= 0) {
if (fn[pos] == '('_L1)
paren--;
else if (fn[pos] == ')'_L1)
paren++;
if (paren == 1 && fn[pos] == ','_L1)
return ' ' + fn.sliced(pos).removeLast().simplified();
pos--;
}
return {};
}
void transformMessageNoLocation(TranslatorMessage &msg, const RecordDirectory &records,
QSet<QString> &ids, Translator &transformedTor, bool labels)
{
if (msg.id().isEmpty()) {
const QString id = records.calculateId(msg);
if (labels)
msg.setLabel(msg.context());
msg.setId(id);
msg.setContext({});
msg.setComment({});
ids.insert(id);
}
transformedTor.append(msg);
}
void transformMessageWithLocation(TranslatorMessage &msg, const RecordDirectory &records,
QSet<QString> &ids, Translator &transformedTor, bool labels)
{
TranslatorMessage::References normalRefs;
TranslatorMessage::References nonsupportedRefs;
for (const TranslatorMessage::Reference &r : msg.allReferences()) {
TranslatorMessage::Reference ur{ r.fileName(),
r.lineNumber()
+ records.addedLines(r.fileName(), r.lineNumber()),
r.startOffset(), r.endOffset() };
if (records.isNonSupported(ur.fileName(), ur.lineNumber()))
nonsupportedRefs.append(ur);
else
normalRefs.append(ur);
}
msg.clearReferences();
if (!nonsupportedRefs.isEmpty()) {
TranslatorMessage newMsg = msg;
newMsg.setReferences(nonsupportedRefs);
transformedTor.append(newMsg);
}
if (QString ctx = msg.context(); !ctx.isEmpty() && !normalRefs.empty()) {
msg.setReferences(normalRefs);
const QString id = records.id(msg);
msg.setId(id);
msg.setContext({});
ids.insert(id);
msg.setComment({});
if (labels)
msg.setLabel(ctx);
transformedTor.append(msg);
} else if (!normalRefs.empty()) {
msg.setReferences(normalRefs);
transformedTor.append(msg);
}
}
bool makeFormIdBased(QStringList &lines, const QString &filename, const QString &label)
{
auto itr = lines.begin();
qsizetype pos;
do // skip comment lines
pos = itr++->indexOf(uiElement);
while (pos < 0 && itr != lines.end());
if (pos < 0) {
printErr("ltext2id: no root element in the "
"ui file %1. Ignoring."_L1.arg(filename));
return false;
}
itr--;
if (!label.isEmpty() && itr->indexOf("label=") < 0)
itr->insert(pos + uiElement.size(), "label=\"" + label + "\" ");
if (itr->indexOf("idbasedtr") < 0)
itr->insert(pos + uiElement.size(), "idbasedtr=\"true\" ");
else
itr->replace("idbasedtr=\"false\"", "idbasedtr=\"true\"");
return true;
}
} // namespace
QT_BEGIN_NAMESPACE
// Static member definitions
const QSet<QString> FileTransformer::cppExtensions{ "c"_L1, "c++"_L1, "cc"_L1, "cpp"_L1,
"cxx"_L1, "ch"_L1, "h"_L1, "h++"_L1,
"hh"_L1, "hpp"_L1, "hxx"_L1 };
const QSet<QString> FileTransformer::otherExtensions{
"jui"_L1, "ui"_L1, "js"_L1, "mjs"_L1, "qml"_L1, // python doesn't have id based
};
FileTransformer::FileTransformer(RecordDirectory &records, bool labels, bool quiet)
: m_records(records), m_labels(labels), m_quiet(quiet)
{
}
void FileTransformer::transformUi()
{
for (const auto &[filename, messages] : m_records.messageLocations().asKeyValueRange()) {
if (filename.endsWith(".ui", Qt::CaseInsensitive)) {
QStringList lines = readLines(filename);
if (lines.empty())
continue;
if (!m_quiet)
printOut("ltext2id: processing source file %1"_L1.arg(filename));
if (m_labels)
makeFormIdBased(lines, filename, (*messages.begin())->context);
for (const std::shared_ptr<MessageItem> &msg : messages) {
QString &line = lines[msg->lineNo - 1];
qsizetype pos = line.indexOf(stringElement);
qsizetype size = stringElement.size();
if (pos < 0 || line.size() < pos + size
|| (line.at(pos + size) != '>' && !line.at(pos + size).isSpace())) {
pos = line.indexOf(stringListElement);
size = stringListElement.size();
}
if (pos < 0 || line.size() < pos + size
|| (line.at(pos + size) != '>' && !line.at(pos + size).isSpace())) {
printErr("ltext2id error: could not find the "
"expected translatable string in %1:%2.\n"_L1.arg(filename)
.arg(msg->lineNo));
m_records.recordError(filename, msg->lineNo,
"please use id %1"_L1.arg(msg->id));
continue;
}
pos += size;
line.insert(pos, " id=\"" + msg->id + "\"");
}
writeLines(filename, lines);
}
}
}
void FileTransformer::transformSources()
{
for (const auto &[filename, messages] : m_records.messageLocations().asKeyValueRange()) {
if (!filename.endsWith(".ui", Qt::CaseInsensitive)) {
if (!m_quiet)
printOut("ltext2id: processing source file %1"_L1.arg(filename));
QFile file(filename);
if (!file.open(QIODevice::ReadOnly)) {
printErr("ltext2id error: failed to open file %1 for reading.\n"_L1.arg(filename));
continue;
}
QTextStream in(&file);
const QString code = in.readAll();
file.close();
QString newCode;
newCode.reserve(code.size());
qsizetype lastPos = 0;
int addedLines = 0;
for (const std::shared_ptr<MessageItem> &msg : messages) {
if (msg->startOffset >= 0 && msg->endOffset > msg->startOffset
&& msg->endOffset <= code.size() && msg->startOffset >= lastPos) {
int lastLinePos = code.lastIndexOf('\n', msg->startOffset) + 1;
if (lastLinePos < lastPos)
lastLinePos = msg->startOffset;
const QString lastLine =
code.sliced(lastLinePos, msg->startOffset - lastLinePos);
const QString indentation = getIndentation(lastLine);
QString fnId = textMetaString(indentation, msg->sourceText);
if (m_labels)
fnId += labelMetaString(indentation, msg->context);
QString fn = code.sliced(msg->startOffset, msg->endOffset - msg->startOffset);
int trFunc = getTrFunction(fn);
QString idBasedFn = idBasedFunc(trFunc, msg->id, getPluralArg(fn, msg->plural));
if (idBasedFn.isEmpty()) {
msg->lineNo += addedLines;
if (trFunc >= 0)
m_records.recordNonSupported(filename, msg->lineNo);
else
m_records.recordError(filename, msg->lineNo,
"Could not detect any translation calls here"_L1);
continue;
}
fnId += lastLine + std::move(idBasedFn);
QString codePiece = code.sliced(lastPos, lastLinePos - lastPos);
if (msg->hasMetaId) {
codePiece.remove(idMetaStringRegex(msg->id));
addedLines--;
}
newCode += codePiece;
newCode += fnId;
addedLines += fnId.count('\n');
m_records.recordAddedLines(filename, msg->lineNo, addedLines);
msg->lineNo += addedLines;
if (const int origMsgLines = fn.count('\n'); origMsgLines) {
addedLines -= origMsgLines;
m_records.recordAddedLines(filename, msg->lineNo - addedLines + 1,
addedLines);
}
lastPos = msg->endOffset;
} else {
m_records.recordError(
filename, msg->lineNo,
QString("Invalid location offsets for the translation call: %1-%2")
.arg(msg->startOffset)
.arg(msg->endOffset));
}
}
newCode += code.sliced(lastPos);
QFile outFile(filename);
if (!outFile.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
printErr("ltext2id error: failed to open file %1 for writing.\n"_L1.arg(filename));
continue;
}
QTextStream out(&outFile);
out << newCode;
outFile.close();
}
}
}
bool FileTransformer::transformTsFiles(const QStringList &translations, bool sortMessages)
{
ConversionData cd;
cd.m_sortMessages = sortMessages;
bool ok = true;
for (const QString &tsFile : std::as_const(translations)) {
if (!m_quiet)
printOut("ltext2id: processing TS file %1"_L1.arg(tsFile));
Translator tor;
tor.load(tsFile, cd, "ts");
if (!cd.errors().empty()) { // timestamp files
cd.clearErrors();
continue;
}
tor.makeFileNamesAbsolute(QFileInfo(tsFile).absoluteDir());
Translator transformedTor;
QSet<QString> ids;
for (qsizetype i = 0; i < tor.messageCount(); i++) {
TranslatorMessage &msg = tor.message(i);
if (const QString filename = msg.fileName();
filename.isEmpty() || !m_records.containsFile(filename))
transformMessageNoLocation(msg, m_records, ids, transformedTor, m_labels);
else
transformMessageWithLocation(msg, m_records, ids, transformedTor, m_labels);
}
transformedTor.save(tsFile, cd, "ts");
if (!cd.errors().empty()) {
printErr(
"ltext2id error: error in processing translation files\n%1"_L1.arg(cd.error()));
ok = false;
}
if (!m_quiet)
printOut("ltext2id: verifying TS file %1"_L1.arg(tsFile));
FileVerifier verifier(m_records, m_quiet);
if (!verifier.verifyTs(tsFile, ids)) {
printErr("ltext2id: verifying TS file %1 failed."_L1.arg(tsFile));
ok = false;
}
}
return ok;
}
void FileTransformer::generateMetaIds()
{
for (const auto &[filename, messages] : m_records.messageLocations().asKeyValueRange()) {
if (!filename.endsWith(".ui", Qt::CaseInsensitive)) {
if (!m_quiet)
printOut("ltext2id: processing source file %1"_L1.arg(filename));
QStringList lines = readLines(filename);
int addedLines = 0;
for (auto itr = messages.cbegin(); itr != messages.cend(); itr++) {
const MessageItem &m = **itr;
QString &line = lines[m.lineNo - 1];
const QString indentation = getIndentation(lines[m.lineNo - 1]);
line = idMetaString(indentation, m.id) + line;
m_records.recordAddedLines(filename, m.lineNo, ++addedLines);
}
writeLines(filename, lines);
}
}
}
bool FileTransformer::updateTsFiles(const QStringList &translations)
{
ConversionData cd;
bool ok = true;
for (const QString &tsFile : std::as_const(translations)) {
if (!m_quiet)
printOut("ltext2id: processing TS file %1"_L1.arg(tsFile));
Translator tor;
tor.load(tsFile, cd, "ts");
if (!cd.errors().empty()) { // timestamp files
cd.clearErrors();
continue;
}
tor.makeFileNamesAbsolute(QFileInfo(tsFile).absoluteDir());
Translator transformedTor;
for (qsizetype i = 0; i < tor.messageCount(); i++) {
TranslatorMessage &msg = tor.message(i);
TranslatorMessage::References refs;
for (const TranslatorMessage::Reference &r : msg.allReferences()) {
if (const QString filename = r.fileName();
!filename.isEmpty() && m_records.containsFile(filename)) {
TranslatorMessage::Reference ur{
filename, r.lineNumber() + m_records.addedLines(filename, r.lineNumber()),
r.startOffset(), r.endOffset()
};
refs.append(ur);
}
}
msg.clearReferences();
msg.setReferences(refs);
msg.setExtra(meta_id_key, m_records.id(msg));
transformedTor.append(msg);
}
transformedTor.save(tsFile, cd, "ts");
if (!cd.errors().empty()) {
printErr(
"ltext2id error: error in processing translation files\n%1"_L1.arg(cd.error()));
ok = false;
}
if (!m_quiet)
printOut("ltext2id: verifying TS file %1"_L1.arg(tsFile));
}
return ok;
}
QT_END_NAMESPACE