Revision control
++ // Copyright (C) 2025 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
++
++ #include "filetransformer.h"
++ #include "fileverifier.h"
++ #include "utils.h"
++ #include "recorddirectory.h"
++ #include <translator.h>
++ #include <trlib/metastrings.h>
++ #include <trlib/trparser.h>
++
++ #include <QFile>
++ #include <QFileInfo>
++ #include <QTextStream>
++ #include <QSet>
++ #include <QRegularExpression>
++
++ using namespace Qt::StringLiterals;
++ using namespace Utils;
++
++ namespace {
++
++ // UI elements for XML processing
++ constexpr QLatin1String uiElement = "<ui "_L1;
++ constexpr QLatin1String stringElement = "<string"_L1;
++ constexpr QLatin1String stringListElement = "<stringlist"_L1;
++
++ QString escapeForCppStringLiteral(const QString &str)
++ {
++ QString result = str;
++ result.replace("\\", "\\\\");
++ result.replace("\"", "\\\"");
++ return result;
++ }
++
++ QString textMetaString(const QString &indentation, const QString &sourceText)
++ {
++ const QStringList lines = sourceText.split('\n');
++ QString metaString;
++ for (int i = 0; i < lines.size(); i++) {
++ metaString += indentation + "//" + MetaStrings::sourceTextAnotation + " \""
++ + escapeForCppStringLiteral(lines[i]);
++ if (i < lines.size() - 1)
++ metaString += "\\n\"\n";
++ else
++ metaString += "\"\n";
++ }
++ return metaString;
++ }
++
++ QString labelMetaString(const QString &indentation, const QString &label)
++ {
++ return indentation + "//" + MetaStrings::labelAnotation + ' ' + label + '\n';
++ }
++
++ QString idMetaString(const QString &indentation, const QString &id)
++ {
++ return indentation + "//" + MetaStrings::extraAnotation + " meta-id " + id + '\n';
++ }
++
++ QRegularExpression idMetaStringRegex(const QString &id)
++ {
++ const QString e = QRegularExpression::escape(id);
++ const QString pat = QStringLiteral(R"(^[ \t]*//~ meta-id\s*%1[ \t]*(?:\r?\n|$))").arg(e);
++ return QRegularExpression(pat, QRegularExpression::MultilineOption);
++ }
++
++ QString idBasedFunc(int trFunc, const QString &id, const QString &pluralArg)
++ {
++ switch (trFunc) {
++ case TrFunctionAliasManager::Function_trUtf8:
++ case TrFunctionAliasManager::Function_tr:
++ case TrFunctionAliasManager::Function_translate:
++ return "qtTrId(\"" + id + "\"" + pluralArg + ")";
++ case TrFunctionAliasManager::Function_qsTr:
++ case TrFunctionAliasManager::Function_qsTranslate:
++ return "qsTrId(\"" + id + "\"" + pluralArg + ")";
++ case TrFunctionAliasManager::Function_QT_TR_NOOP:
++ case TrFunctionAliasManager::Function_QT_TR_NOOP_UTF8:
++ case TrFunctionAliasManager::Function_QT_TRANSLATE_NOOP:
++ case TrFunctionAliasManager::Function_QT_TRANSLATE_NOOP_UTF8:
++ return "QT_TRID_NOOP(\"" + id + "\")";
++ case TrFunctionAliasManager::Function_QT_TR_N_NOOP:
++ case TrFunctionAliasManager::Function_QT_TRANSLATE_N_NOOP:
++ return "QT_TRID_N_NOOP(\"" + id + "\")";
++ default:
++ return {};
++ }
++ }
++
++ int getTrFunction(const QString &expr)
++ {
++ int i = 0;
++ int depth = 0;
++ QString trFunc;
++ while (i < expr.size()) {
++ if (QChar c = expr[i++]; (c.isLetterOrNumber() || c == '_'_L1) && depth == 0)
++ trFunc += c;
++ else {
++ while (c.isSpace() && i < expr.size())
++ c = expr[i++];
++
++ if (c == '(') {
++ if (depth == 0)
++ if (int trFuncId = trFunctionAliasManager.trFunctionByName(trFunc);
++ trFuncId >= 0)
++ return trFuncId;
++ depth++;
++ } else if (c == ')'_L1)
++ depth--;
++ else if (c == "\""_L1 || c == "'"_L1) {
++ const QChar quotation = c;
++ while (i < expr.size()) {
++ c = expr[i++];
++ if (c == '\\'_L1)
++ i++;
++ else if (c == quotation)
++ break;
++ }
++ }
++ trFunc.clear();
++ }
++ }
++ return -1;
++ }
++
++ QString getPluralArg(const QString &fn, bool plural)
++ {
++ if (!plural)
++ return {};
++ int paren = 1;
++ int pos = fn.size() - 2;
++ while (paren > 0 && pos >= 0) {
++ if (fn[pos] == '('_L1)
++ paren--;
++ else if (fn[pos] == ')'_L1)
++ paren++;
++ if (paren == 1 && fn[pos] == ','_L1)
++ return ' ' + fn.sliced(pos).removeLast().simplified();
++ pos--;
++ }
++ return {};
++ }
++
++ void transformMessageNoLocation(TranslatorMessage &msg, const RecordDirectory &records,
++ QSet<QString> &ids, Translator &transformedTor, bool labels)
++ {
++ if (msg.id().isEmpty()) {
++ const QString id = records.calculateId(msg);
++ if (labels)
++ msg.setLabel(msg.context());
++ msg.setId(id);
++ msg.setContext({});
++ msg.setComment({});
++ ids.insert(id);
++ }
++ transformedTor.append(msg);
++ }
++
++ void transformMessageWithLocation(TranslatorMessage &msg, const RecordDirectory &records,
++ QSet<QString> &ids, Translator &transformedTor, bool labels)
++ {
++ TranslatorMessage::References normalRefs;
++ TranslatorMessage::References nonsupportedRefs;
++ for (const TranslatorMessage::Reference &r : msg.allReferences()) {
++ TranslatorMessage::Reference ur{ r.fileName(),
++ r.lineNumber()
++ + records.addedLines(r.fileName(), r.lineNumber()),
++ r.startOffset(), r.endOffset() };
++ if (records.isNonSupported(ur.fileName(), ur.lineNumber()))
++ nonsupportedRefs.append(ur);
++ else
++ normalRefs.append(ur);
++ }
++ msg.clearReferences();
++ if (!nonsupportedRefs.isEmpty()) {
++ TranslatorMessage newMsg = msg;
++ newMsg.setReferences(nonsupportedRefs);
++ transformedTor.append(newMsg);
++ }
++ if (QString ctx = msg.context(); !ctx.isEmpty() && !normalRefs.empty()) {
++ msg.setReferences(normalRefs);
++ const QString id = records.id(msg);
++ msg.setId(id);
++ msg.setContext({});
++ ids.insert(id);
++ msg.setComment({});
++ if (labels)
++ msg.setLabel(ctx);
++ transformedTor.append(msg);
++ } else if (!normalRefs.empty()) {
++ msg.setReferences(normalRefs);
++ transformedTor.append(msg);
++ }
++ }
++
++ bool makeFormIdBased(QStringList &lines, const QString &filename, const QString &label)
++ {
++ auto itr = lines.begin();
++ qsizetype pos;
++ do // skip comment lines
++ pos = itr++->indexOf(uiElement);
++ while (pos < 0 && itr != lines.end());
++
++ if (pos < 0) {
++ printErr("ltext2id: no root element in the "
++ "ui file %1. Ignoring."_L1.arg(filename));
++ return false;
++ }
++ itr--;
++
++ if (!label.isEmpty() && itr->indexOf("label=") < 0)
++ itr->insert(pos + uiElement.size(), "label=\"" + label + "\" ");
++
++ if (itr->indexOf("idbasedtr") < 0)
++ itr->insert(pos + uiElement.size(), "idbasedtr=\"true\" ");
++ else
++ itr->replace("idbasedtr=\"false\"", "idbasedtr=\"true\"");
++
++ return true;
++ }
++ } // namespace
++
++ QT_BEGIN_NAMESPACE
++
++ // Static member definitions
++ const QSet<QString> FileTransformer::cppExtensions{ "c"_L1, "c++"_L1, "cc"_L1, "cpp"_L1,
++ "cxx"_L1, "ch"_L1, "h"_L1, "h++"_L1,
++ "hh"_L1, "hpp"_L1, "hxx"_L1 };
++
++ const QSet<QString> FileTransformer::otherExtensions{
++ "jui"_L1, "ui"_L1, "js"_L1, "mjs"_L1, "qml"_L1, // python doesn't have id based
++ };
++
++ FileTransformer::FileTransformer(RecordDirectory &records, bool labels, bool quiet)
++ : m_records(records), m_labels(labels), m_quiet(quiet)
++ {
++ }
++
++ void FileTransformer::transformUi()
++ {
++ for (const auto &[filename, messages] : m_records.messageLocations().asKeyValueRange()) {
++ if (filename.endsWith(".ui", Qt::CaseInsensitive)) {
++ QStringList lines = readLines(filename);
++ if (lines.empty())
++ continue;
++
++ if (!m_quiet)
++ printOut("ltext2id: processing source file %1"_L1.arg(filename));
++ if (m_labels)
++ makeFormIdBased(lines, filename, (*messages.begin())->context);
++
++ for (const std::shared_ptr<MessageItem> &msg : messages) {
++ QString &line = lines[msg->lineNo - 1];
++ qsizetype pos = line.indexOf(stringElement);
++ qsizetype size = stringElement.size();
++ if (pos < 0 || line.size() < pos + size
++ || (line.at(pos + size) != '>' && !line.at(pos + size).isSpace())) {
++ pos = line.indexOf(stringListElement);
++ size = stringListElement.size();
++ }
++ if (pos < 0 || line.size() < pos + size
++ || (line.at(pos + size) != '>' && !line.at(pos + size).isSpace())) {
++ printErr("ltext2id error: could not find the "
++ "expected translatable string in %1:%2.\n"_L1.arg(filename)
++ .arg(msg->lineNo));
++ m_records.recordError(filename, msg->lineNo,
++ "please use id %1"_L1.arg(msg->id));
++ continue;
++ }
++ pos += size;
++ line.insert(pos, " id=\"" + msg->id + "\"");
++ }
++ writeLines(filename, lines);
++ }
++ }
++ }
++
++ void FileTransformer::transformSources()
++ {
++ for (const auto &[filename, messages] : m_records.messageLocations().asKeyValueRange()) {
++ if (!filename.endsWith(".ui", Qt::CaseInsensitive)) {
++ if (!m_quiet)
++ printOut("ltext2id: processing source file %1"_L1.arg(filename));
++
++ QFile file(filename);
++ if (!file.open(QIODevice::ReadOnly)) {
++ printErr("ltext2id error: failed to open file %1 for reading.\n"_L1.arg(filename));
++ continue;
++ }
++ QTextStream in(&file);
++ const QString code = in.readAll();
++ file.close();
++
++ QString newCode;
++ newCode.reserve(code.size());
++ qsizetype lastPos = 0;
++ int addedLines = 0;
++ for (const std::shared_ptr<MessageItem> &msg : messages) {
++ if (msg->startOffset >= 0 && msg->endOffset > msg->startOffset
++ && msg->endOffset <= code.size() && msg->startOffset >= lastPos) {
++ int lastLinePos = code.lastIndexOf('\n', msg->startOffset) + 1;
++ if (lastLinePos < lastPos)
++ lastLinePos = msg->startOffset;
++ const QString lastLine =
++ code.sliced(lastLinePos, msg->startOffset - lastLinePos);
++
++ const QString indentation = getIndentation(lastLine);
++ QString fnId = textMetaString(indentation, msg->sourceText);
++ if (m_labels)
++ fnId += labelMetaString(indentation, msg->context);
++ QString fn = code.sliced(msg->startOffset, msg->endOffset - msg->startOffset);
++ int trFunc = getTrFunction(fn);
++ QString idBasedFn = idBasedFunc(trFunc, msg->id, getPluralArg(fn, msg->plural));
++ if (idBasedFn.isEmpty()) {
++ msg->lineNo += addedLines;
++ if (trFunc >= 0)
++ m_records.recordNonSupported(filename, msg->lineNo);
++ else
++ m_records.recordError(filename, msg->lineNo,
++ "Could not detect any translation calls here"_L1);
++ continue;
++ }
++ fnId += lastLine + std::move(idBasedFn);
++
++ QString codePiece = code.sliced(lastPos, lastLinePos - lastPos);
++ if (msg->hasMetaId) {
++ codePiece.remove(idMetaStringRegex(msg->id));
++ addedLines--;
++ }
++ newCode += codePiece;
++ newCode += fnId;
++
++ addedLines += fnId.count('\n');
++ m_records.recordAddedLines(filename, msg->lineNo, addedLines);
++ msg->lineNo += addedLines;
++ if (const int origMsgLines = fn.count('\n'); origMsgLines) {
++ addedLines -= origMsgLines;
++ m_records.recordAddedLines(filename, msg->lineNo - addedLines + 1,
++ addedLines);
++ }
++ lastPos = msg->endOffset;
++ } else {
++ m_records.recordError(
++ filename, msg->lineNo,
++ QString("Invalid location offsets for the translation call: %1-%2")
++ .arg(msg->startOffset)
++ .arg(msg->endOffset));
++ }
++ }
++ newCode += code.sliced(lastPos);
++
++ QFile outFile(filename);
++ if (!outFile.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
++ printErr("ltext2id error: failed to open file %1 for writing.\n"_L1.arg(filename));
++ continue;
++ }
++ QTextStream out(&outFile);
++ out << newCode;
++ outFile.close();
++ }
++ }
++ }
++
++ bool FileTransformer::transformTsFiles(const QStringList &translations, bool sortMessages)
++ {
++ ConversionData cd;
++ cd.m_sortMessages = sortMessages;
++ bool ok = true;
++ for (const QString &tsFile : std::as_const(translations)) {
++ if (!m_quiet)
++ printOut("ltext2id: processing TS file %1"_L1.arg(tsFile));
++ Translator tor;
++ tor.load(tsFile, cd, "ts");
++ if (!cd.errors().empty()) { // timestamp files
++ cd.clearErrors();
++ continue;
++ }
++ tor.makeFileNamesAbsolute(QFileInfo(tsFile).absoluteDir());
++
++ Translator transformedTor;
++ QSet<QString> ids;
++ for (qsizetype i = 0; i < tor.messageCount(); i++) {
++ TranslatorMessage &msg = tor.message(i);
++ if (const QString filename = msg.fileName();
++ filename.isEmpty() || !m_records.containsFile(filename))
++ transformMessageNoLocation(msg, m_records, ids, transformedTor, m_labels);
++ else
++ transformMessageWithLocation(msg, m_records, ids, transformedTor, m_labels);
++ }
++ transformedTor.save(tsFile, cd, "ts");
++ if (!cd.errors().empty()) {
++ printErr(
++ "ltext2id error: error in processing translation files\n%1"_L1.arg(cd.error()));
++ ok = false;
++ }
++
++ if (!m_quiet)
++ printOut("ltext2id: verifying TS file %1"_L1.arg(tsFile));
++
++ FileVerifier verifier(m_records, m_quiet);
++ if (!verifier.verifyTs(tsFile, ids)) {
++ printErr("ltext2id: verifying TS file %1 failed."_L1.arg(tsFile));
++ ok = false;
++ }
++ }
++ return ok;
++ }
++
++ void FileTransformer::generateMetaIds()
++ {
++ for (const auto &[filename, messages] : m_records.messageLocations().asKeyValueRange()) {
++ if (!filename.endsWith(".ui", Qt::CaseInsensitive)) {
++ if (!m_quiet)
++ printOut("ltext2id: processing source file %1"_L1.arg(filename));
++
++ QStringList lines = readLines(filename);
++ int addedLines = 0;
++ for (auto itr = messages.cbegin(); itr != messages.cend(); itr++) {
++ const MessageItem &m = **itr;
++ QString &line = lines[m.lineNo - 1];
++ const QString indentation = getIndentation(lines[m.lineNo - 1]);
++ line = idMetaString(indentation, m.id) + line;
++ m_records.recordAddedLines(filename, m.lineNo, ++addedLines);
++ }
++ writeLines(filename, lines);
++ }
++ }
++ }
++
++ bool FileTransformer::updateTsFiles(const QStringList &translations)
++ {
++ ConversionData cd;
++ bool ok = true;
++ for (const QString &tsFile : std::as_const(translations)) {
++ if (!m_quiet)
++ printOut("ltext2id: processing TS file %1"_L1.arg(tsFile));
++ Translator tor;
++ tor.load(tsFile, cd, "ts");
++ if (!cd.errors().empty()) { // timestamp files
++ cd.clearErrors();
++ continue;
++ }
++ tor.makeFileNamesAbsolute(QFileInfo(tsFile).absoluteDir());
++ Translator transformedTor;
++
++ for (qsizetype i = 0; i < tor.messageCount(); i++) {
++ TranslatorMessage &msg = tor.message(i);
++ TranslatorMessage::References refs;
++ for (const TranslatorMessage::Reference &r : msg.allReferences()) {
++ if (const QString filename = r.fileName();
++ !filename.isEmpty() && m_records.containsFile(filename)) {
++ TranslatorMessage::Reference ur{
++ filename, r.lineNumber() + m_records.addedLines(filename, r.lineNumber()),
++ r.startOffset(), r.endOffset()
++ };
++ refs.append(ur);
++ }
++ }
++ msg.clearReferences();
++ msg.setReferences(refs);
++ msg.setExtra(meta_id_key, m_records.id(msg));
++ transformedTor.append(msg);
++ }
++
++ transformedTor.save(tsFile, cd, "ts");
++ if (!cd.errors().empty()) {
++ printErr(
++ "ltext2id error: error in processing translation files\n%1"_L1.arg(cd.error()));
++ ok = false;
++ }
++
++ if (!m_quiet)
++ printOut("ltext2id: verifying TS file %1"_L1.arg(tsFile));
++ }
++ return ok;
++ }
++
++ QT_END_NAMESPACE