qt/qttools/tests/auto/linguist/lconvert/tst_lconvert.cpp

Revision control

++ // Copyright (C) 2016 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
++
++ #include <QFile>
++ #include <QLibraryInfo>
++ #include <QProcess>
++ #include <QTemporaryDir>
++ #include <QTest>
++ #include <QTextStream>
++
++ using namespace Qt::Literals::StringLiterals;
++
++ class tst_lconvert : public QObject
++ {
++ Q_OBJECT
++
++ public:
++ tst_lconvert()
++ : dataDir(QFINDTESTDATA("data/"))
++ , lconvert(QLibraryInfo::path(QLibraryInfo::BinariesPath) + "/lconvert") {}
++
++ private slots:
++ void initTestCase();
++ void readverifies_data();
++ void readverifies();
++ void converts_data();
++ void converts();
++ void roundtrips_data();
++ void roundtrips();
++ void chains_data();
++ void chains();
++ void merge();
++ void qmRoundtripPreservesBytes();
++ void tsLineEndings_data();
++ void tsLineEndings();
++
++ private:
++ void doWait(QProcess *cvt, int stage);
++ void doCompare(QIODevice *actual, const QString &expectedFn);
++ void verifyReadFail(const QString &fn);
++ // args can be empty or have one element less than stations
++ void convertChain(const QString &inFileName, const QString &outFileName,
++ const QStringList &stations, const QList<QStringList> &args);
++ void convertRoundtrip(const QString &fileName, const QStringList &stations,
++ const QList<QStringList> &args);
++
++ QString dataDir;
++ QString lconvert;
++ };
++
++ static void writePoMessages(QTextStream &out, int num, const QString &str)
++ {
++ for (int i = 0; i < (1 << num); ++i) {
++ out << "\n";
++ out << "msgid \"singular " << str << " " << i << "\"\n";
++ out << "msgid_plural \"plural " << str << " " << i << "\"\n";
++ for (int j = 0; j < num; ++j) {
++ QString tr;
++ if ((i & (1 << j)) == 0)
++ tr = u"translated %1 %2 %3"_s.arg(str).arg(i).arg(j);
++ out << "msgstr[" << j << "] \"" << tr << "\"\n";
++ }
++ }
++ }
++
++ static void createPluralPoFile(const QString &outdir, int num, const QString &lang,
++ const QString &pluralForms)
++ {
++ QString filename = outdir + u"plural-%1.po"_s.arg(num);
++ QFile file(filename);
++ if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) {
++ qWarning() << "Cannot write file in" << outdir;
++ return;
++ }
++
++ QTextStream out(&file);
++ out << R"(msgid ""
++ msgstr ""
++ "MIME-Version: 1.0\n"
++ "Content-Type: text/plain; charset=UTF-8\n"
++ "Content-Transfer-Encoding: 8bit\n"
++ "X-FooBar: yup\n"
++ "X-Language: )" << lang << R"(\n"
++ "Plural-Forms: )" << pluralForms << "\\n\"\n";
++
++ writePoMessages(out, num, "one"_L1);
++ writePoMessages(out, num, "two"_L1);
++ writePoMessages(out, num, "three"_L1);
++ writePoMessages(out, num, "four"_L1);
++ }
++
++ void tst_lconvert::initTestCase()
++ {
++ if (!QFile::exists(dataDir + QLatin1String("plural-1.po"))) {
++ createPluralPoFile(dataDir, 1, u"zh_CN"_s, u"nplurals=1; plural=0;"_s);
++ createPluralPoFile(dataDir, 2, u"de_DE"_s, u"nplurals=2; plural=(n != 1);"_s);
++ createPluralPoFile(dataDir, 3, u"pl_PL"_s,
++ u"nplurals=3; plural=(n==1 ? 0 : n%10>=2 && "_s
++ u"n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);"_s);
++ }
++ QVERIFY(QFile::exists(dataDir + QLatin1String("plural-1.po")));
++ }
++
++ void tst_lconvert::doWait(QProcess *cvt, int stage)
++ {
++ if (QTest::currentTestFailed()) {
++ cvt->kill();
++ cvt->waitForFinished();
++ } else {
++ QVERIFY2(cvt->waitForFinished(3000),
++ qPrintable(QString("Process %1 hung").arg(stage)));
++ QVERIFY2(cvt->exitStatus() == QProcess::NormalExit,
++ qPrintable(QString("Process %1 crashed").arg(stage)));
++ QVERIFY2(cvt->exitCode() == 0,
++ qPrintable(QString("Process %1 exited with status %2. Errors:\n%3")
++ .arg(stage).arg(cvt->exitCode())
++ .arg(QString::fromUtf8(cvt->readAllStandardError()))));
++ }
++ }
++
++ void tst_lconvert::doCompare(QIODevice *actualDev, const QString &expectedFn)
++ {
++ QList<QByteArray> actual = actualDev->readAll().split('\n');
++
++ QFile file(expectedFn);
++ QVERIFY(file.open(QIODevice::ReadOnly | QIODevice::Text));
++ QList<QByteArray> expected = file.readAll().split('\n');
++
++ int i = 0, ei = expected.size(), gi = actual.size();
++ for (; ; i++) {
++ if (i == gi) {
++ if (i == ei)
++ return;
++ gi = 0;
++ break;
++ } else if (i == ei) {
++ ei = 0;
++ break;
++ } else if (actual.at(i) != expected.at(i)) {
++ while ((ei - 1) >= i && (gi - 1) >= i && actual.at(gi - 1) == expected.at(ei - 1))
++ ei--, gi--;
++ break;
++ }
++ }
++ QByteArray diff;
++ for (int j = qMax(0, i - 3); j < i; j++)
++ diff += expected.at(j) + '\n';
++ diff += "<<<<<<< got\n";
++ for (int j = i; j < gi; j++) {
++ diff += actual.at(j) + '\n';
++ if (j >= i + 5) {
++ diff += "...\n";
++ break;
++ }
++ }
++ diff += "=========\n";
++ for (int j = i; j < ei; j++) {
++ diff += expected.at(j) + '\n';
++ if (j >= i + 5) {
++ diff += "...\n";
++ break;
++ }
++ }
++ diff += ">>>>>>> expected\n";
++ for (int j = ei; j < qMin(ei + 3, expected.size()); j++)
++ diff += expected.at(j) + '\n';
++ QFAIL(qPrintable("Output for " + expectedFn + " does not meet expectations:\n" + diff));
++ }
++
++ void tst_lconvert::verifyReadFail(const QString &fn)
++ {
++ QProcess cvt;
++ cvt.start(lconvert, QStringList() << (dataDir + fn));
++ QVERIFY2(cvt.waitForStarted(), qPrintable(cvt.errorString()));
++ QVERIFY(cvt.waitForFinished(10000));
++ QVERIFY(cvt.exitStatus() == QProcess::NormalExit);
++ QVERIFY2(cvt.exitCode() == 2, "Accepted invalid input");
++ }
++
++ void tst_lconvert::convertChain(const QString &_inFileName, const QString &_outFileName,
++ const QStringList &stations, const QList<QStringList> &argList)
++ {
++ QList<QProcess *> cvts;
++
++ QString fileName = dataDir + _inFileName;
++ QString outFileName = dataDir + _outFileName;
++
++ for (int i = 0; i < stations.size() - 1; i++) {
++ QProcess *cvt = new QProcess(this);
++ if (cvts.isEmpty())
++ cvt->setStandardInputFile(fileName);
++ else
++ cvts.last()->setStandardOutputProcess(cvt);
++ cvts.append(cvt);
++ }
++ for (int i = 0; i < stations.size() - 1; i++) {
++ QStringList args;
++ if (!argList.isEmpty())
++ args += argList[i];
++ args << "-if" << stations[i] << "-i" << "-" << "-of" << stations[i + 1];
++ cvts.at(i)->start(lconvert, args, QIODevice::ReadWrite | QIODevice::Text);
++ }
++ for (QProcess *cvt : std::as_const(cvts))
++ QVERIFY2(cvt->waitForStarted(), qPrintable(cvt->errorString()));
++ int st = 0;
++ for (QProcess *cvt : std::as_const(cvts))
++ doWait(cvt, ++st);
++
++ if (!QTest::currentTestFailed())
++ doCompare(cvts.last(), outFileName);
++
++ qDeleteAll(cvts);
++ }
++
++ void tst_lconvert::convertRoundtrip(const QString &_fileName, const QStringList &stations,
++ const QList<QStringList> &argList)
++ {
++ convertChain(_fileName, _fileName, stations, argList);
++ }
++
++ void tst_lconvert::readverifies_data()
++ {
++ QTest::addColumn<QString>("fileName");
++ QTest::addColumn<QString>("format");
++
++ QTest::newRow("ts") << "test20.ts" << "ts";
++ QTest::newRow("empty comment") << "test-empty-comment.po" << "po";
++ QTest::newRow("translator comment") << "test-translator-comment.po" << "po";
++ QTest::newRow("developer comment") << "test-developer-comment.po" << "po";
++ QTest::newRow("kde context") << "test-kde-ctxt.po" << "po";
++ QTest::newRow("kde fuzzy") << "test-kde-fuzzy.po" << "po";
++ QTest::newRow("kde plurals") << "test-kde-plurals.po" << "po";
++ QTest::newRow("kde multiline") << "test-kde-multiline.po" << "po";
++ QTest::newRow("po linewrapping") << "wrapping.po" << "po";
++ QTest::newRow("relative locations") << "relative.ts" << "ts";
++ QTest::newRow("message ids") << "msgid.ts" << "ts";
++ QTest::newRow("length variants") << "variants.ts" << "ts";
++ QTest::newRow("qph") << "phrasebook.qph" << "qph";
++ }
++
++ void tst_lconvert::readverifies()
++ {
++ QFETCH(QString, fileName);
++ QFETCH(QString, format);
++
++ convertRoundtrip(fileName, QStringList() << format << format, QList<QStringList>());
++ }
++
++ void tst_lconvert::converts_data()
++ {
++ QTest::addColumn<QString>("inFileName");
++ QTest::addColumn<QString>("outFileName");
++ QTest::addColumn<QString>("format");
++
++ QTest::newRow("broken utf8") << "test-broken-utf8.po" << "test-broken-utf8.po.out" << "po";
++ QTest::newRow("line joins") << "test-slurp.po" << "test-slurp.po.out" << "po";
++ QTest::newRow("escapes") << "test-escapes.po" << "test-escapes.po.out" << "po";
++ QTest::newRow("xlf seg") << "test-trans_seg.xlf" << "test-trans_seg.ts.out" << "ts";
++ }
++
++ void tst_lconvert::converts()
++ {
++ QFETCH(QString, inFileName);
++ QFETCH(QString, outFileName);
++ QFETCH(QString, format);
++
++ QString outFileNameFq = dataDir + outFileName;
++
++ QProcess cvt;
++ cvt.start(lconvert,
++ QStringList() << "-i" << (dataDir + inFileName) << "-of" << format,
++ QIODevice::ReadWrite | QIODevice::Text);
++ QVERIFY2(cvt.waitForStarted(), qPrintable(cvt.errorString()));
++ doWait(&cvt, 0);
++ if (QTest::currentTestFailed())
++ return;
++
++ doCompare(&cvt, outFileNameFq);
++ }
++
++ Q_DECLARE_METATYPE(QList<QStringList>);
++
++ void tst_lconvert::chains_data()
++ {
++ QTest::addColumn<QString>("inFileName");
++ QTest::addColumn<QString>("outFileName");
++ QTest::addColumn<QStringList>("stations");
++ QTest::addColumn<QList<QStringList> >("args");
++
++ QTest::newRow("no-untranslated") << "untranslated.ts" << "untranslated.ts.out"
++ << QStringList({"ts", "ts"})
++ << QList<QStringList>({QStringList("-no-untranslated")});
++ }
++
++ void tst_lconvert::chains()
++ {
++ QFETCH(QString, inFileName);
++ QFETCH(QString, outFileName);
++ QFETCH(QStringList, stations);
++ QFETCH(QList<QStringList>, args);
++
++ convertChain(inFileName, outFileName, stations, args);
++ }
++
++ void tst_lconvert::roundtrips_data()
++ {
++ QTest::addColumn<QString>("fileName");
++ QTest::addColumn<QStringList>("stations");
++ QTest::addColumn<QList<QStringList> >("args");
++
++ QStringList poTsPo; poTsPo << "po" << "ts" << "po";
++ QStringList poXlfPo; poXlfPo << "po" << "xlf" << "po";
++ QStringList tsPoTs; tsPoTs << "ts" << "po" << "ts";
++ QStringList tsXlfTs; tsXlfTs << "ts" << "xlf" << "ts";
++ QStringList tsQmTs; tsQmTs << "ts" << "qm" << "ts";
++ QStringList qmTsQm; qmTsQm << "qm" << "ts" << "qm";
++
++ QList<QStringList> noArgs;
++ QList<QStringList> filterPoArgs; filterPoArgs << QStringList() << (QStringList() << "-drop-tag" << "po:*");
++ QList<QStringList> outDeArgs; outDeArgs << QStringList() << (QStringList() << "-target-language" << "de");
++ QList<QStringList> outCnArgs; outCnArgs << QStringList() << (QStringList() << "-target-language" << "cn");
++
++ QTest::newRow("po-ts-po (translator comment)") << "test-translator-comment.po" << poTsPo << noArgs;
++ QTest::newRow("po-xliff-po (translator comment)") << "test-translator-comment.po" << poXlfPo << noArgs;
++ QTest::newRow("po-ts-po (developer comment)") << "test-developer-comment.po" << poTsPo << noArgs;
++ QTest::newRow("po-xliff-po (developer comment)") << "test-developer-comment.po" << poXlfPo << noArgs;
++
++ QTest::newRow("ts20-po-ts20") << "test20.ts" << tsPoTs << filterPoArgs;
++ QTest::newRow("po-ts-po (de)") << "test1-de.po" << poTsPo << noArgs;
++ QTest::newRow("po-ts-po (cn)") << "test1-cn.po" << poTsPo << noArgs;
++ QTest::newRow("po-xliff-po (de)") << "test1-de.po" << poXlfPo << noArgs;
++ QTest::newRow("po-xliff-po (cn)") << "test1-cn.po" << poXlfPo << noArgs;
++
++ QTest::newRow("po-ts-po (singular)") << "singular.po" << poTsPo << noArgs;
++ QTest::newRow("po-ts-po (plural-1)") << "plural-1.po" << poTsPo << noArgs;
++ QTest::newRow("po-ts-po (plural-2)") << "plural-2.po" << poTsPo << noArgs;
++ QTest::newRow("po-ts-po (plural-3)") << "plural-3.po" << poTsPo << noArgs;
++ QTest::newRow("po-xliff-po (singular)") << "singular.po" << poXlfPo << noArgs;
++ QTest::newRow("po-xliff-po (plural-1)") << "plural-1.po" << poXlfPo << noArgs;
++ QTest::newRow("po-xliff-po (plural-2)") << "plural-2.po" << poXlfPo << noArgs;
++ QTest::newRow("po-xliff-po (plural-3)") << "plural-3.po" << poXlfPo << noArgs;
++
++ QTest::newRow("po-ts-po (references)") << "test-refs.po" << poTsPo << noArgs;
++
++ QTest::newRow("ts-qm-ts (plurals-de)") << "plurals-de.ts" << tsQmTs << outDeArgs;
++ QTest::newRow("ts-qm-ts (plurals-cn)") << "plurals-cn.ts" << tsQmTs << outCnArgs;
++ QTest::newRow("ts-qm-ts (variants)") << "variants.ts" << tsQmTs << outDeArgs;
++ QTest::newRow("ts-po-ts (msgid)") << "msgid.ts" << tsPoTs << noArgs;
++ QTest::newRow("ts-xliff-ts (msgid)") << "msgid.ts" << tsXlfTs << noArgs;
++ QTest::newRow("ts-qm-ts (emptymsg)") << "emptymsg.ts" << tsQmTs << noArgs;
++ QTest::newRow("ts-xliff-ts (emptymsg)") << "emptymsg.ts" << tsXlfTs << noArgs;
++
++ QTest::newRow("ts-po-ts (endless loop)") << "endless-po-loop.ts" << tsPoTs << noArgs;
++ QTest::newRow("ts-qm-ts (whitespace)") << "whitespace.ts" << tsQmTs << noArgs;
++ QTest::newRow("qm-ts-qm (untranslated)") << "untranslated.qm" << qmTsQm << noArgs;
++ }
++
++ void tst_lconvert::roundtrips()
++ {
++ QFETCH(QString, fileName);
++ QFETCH(QStringList, stations);
++ QFETCH(QList<QStringList>, args);
++
++ convertRoundtrip(fileName, stations, args);
++ }
++
++ void tst_lconvert::merge()
++ {
++ QProcess cvt;
++ QStringList args;
++ args << (dataDir + "idxmerge.ts") << (dataDir + "idxmerge-add.ts");
++ cvt.start(lconvert, args, QIODevice::ReadWrite | QIODevice::Text);
++ doWait(&cvt, 1);
++ if (!QTest::currentTestFailed())
++ doCompare(&cvt, dataDir + "idxmerge.ts.out");
++ }
++
++ // Regression: Translator::save() previously applied QIODevice::Text on Windows
++ // to all output formats. For binary formats (.qm), every \n byte was then
++ // expanded to \r\n whenever the destination already existed and contained
++ // \r\n bytes in its first 4096 bytes (which the CRLF-preservation heuristic
++ // reads as "destination uses CRLF"). The result was a corrupted .qm that
++ // QTranslator silently rejected at load time.
++ void tst_lconvert::qmRoundtripPreservesBytes()
++ {
++ QTemporaryDir tmp;
++ QVERIFY(tmp.isValid());
++
++ // Generate a .qm whose first 4096 bytes contain a \r\n byte pair. The
++ // &#xD;&#xA; entities in the translation survive into the binary output.
++ const char seedTs[] = R"(<?xml version="1.0" encoding="utf-8"?>
++ <!DOCTYPE TS>
++ <TS version="2.1" language="de_DE">
++ <context>
++ <name>X</name>
++ <message>
++ <source>a&#xD;&#xA;b</source>
++ <translation>x&#xD;&#xA;y</translation>
++ </message>
++ </context>
++ </TS>
++ )";
++ const QString seed = tmp.filePath("seed.ts"_L1);
++ {
++ QFile f(seed);
++ QVERIFY2(f.open(QIODevice::WriteOnly), qPrintable(f.errorString()));
++ const QByteArray bytes = QByteArray(seedTs);
++ QCOMPARE(f.write(bytes), bytes.size());
++ }
++
++ const QString input = tmp.filePath("input.qm"_L1);
++ QProcess gen;
++ gen.start(lconvert, QStringList{ seed, "-of"_L1, "qm"_L1, "-o"_L1, input });
++ QVERIFY2(gen.waitForStarted(), qPrintable(gen.errorString()));
++ doWait(&gen, 0);
++ if (QTest::currentTestFailed())
++ return;
++
++ QByteArray expected;
++ {
++ QFile f(input);
++ QVERIFY2(f.open(QIODevice::ReadOnly), qPrintable(f.errorString()));
++ expected = f.readAll();
++ }
++ QVERIFY(!expected.isEmpty());
++ if (!expected.left(4096).contains("\r\n"))
++ QSKIP("Generated .qm contains no CRLF bytes; cannot exercise regression");
++
++ // Pre-populate the destination so usesCRLF() would observe CRLF bytes.
++ const QString output = tmp.filePath("output.qm"_L1);
++ QVERIFY(QFile::copy(input, output));
++
++ QProcess cvt;
++ cvt.start(lconvert, QStringList{ input, "-o"_L1, output });
++ QVERIFY2(cvt.waitForStarted(), qPrintable(cvt.errorString()));
++ doWait(&cvt, 1);
++ if (QTest::currentTestFailed())
++ return;
++
++ QByteArray actual;
++ {
++ QFile f(output);
++ QVERIFY2(f.open(QIODevice::ReadOnly), qPrintable(f.errorString()));
++ actual = f.readAll();
++ }
++ QCOMPARE(actual, expected);
++ }
++
++ void tst_lconvert::tsLineEndings_data()
++ {
++ QTest::addColumn<QByteArray>("seedLineEnding"); // "" = no destination, "\n" or "\r\n"
++ QTest::addColumn<bool>("expectCRLF");
++
++ #ifdef Q_OS_WIN
++ QTest::newRow("preserve CRLF") << QByteArray("\r\n") << true;
++ #endif
++ QTest::newRow("preserve LF") << QByteArray("\n") << false;
++ QTest::newRow("new file defaults to LF") << QByteArray() << false;
++ }
++
++ void tst_lconvert::tsLineEndings()
++ {
++ QFETCH(QByteArray, seedLineEnding);
++ QFETCH(bool, expectCRLF);
++
++ QTemporaryDir tmp;
++ QVERIFY(tmp.isValid());
++
++ QByteArray src;
++ {
++ QFile f(dataDir + "untranslated.ts"_L1);
++ QVERIFY2(f.open(QIODevice::ReadOnly), qPrintable(f.errorString()));
++ src = f.readAll();
++ }
++ QVERIFY(!src.isEmpty());
++ src.replace("\r\n", "\n");
++
++ const QString input = tmp.filePath("input.ts"_L1);
++ {
++ QFile f(input);
++ QVERIFY2(f.open(QIODevice::WriteOnly), qPrintable(f.errorString()));
++ QCOMPARE(f.write(src), src.size());
++ }
++
++ const QString output = tmp.filePath("output.ts"_L1);
++ if (!seedLineEnding.isEmpty()) {
++ QByteArray seed = src;
++ if (seedLineEnding == "\r\n")
++ seed.replace("\n", "\r\n");
++ QFile f(output);
++ QVERIFY2(f.open(QIODevice::WriteOnly), qPrintable(f.errorString()));
++ QCOMPARE(f.write(seed), seed.size());
++ } else {
++ QVERIFY(!QFile::exists(output));
++ }
++
++ QProcess cvt;
++ cvt.start(lconvert, QStringList{ input, "-o"_L1, output });
++ QVERIFY2(cvt.waitForStarted(), qPrintable(cvt.errorString()));
++ doWait(&cvt, 0);
++ if (QTest::currentTestFailed())
++ return;
++
++ QByteArray bytes;
++ {
++ QFile f(output);
++ QVERIFY2(f.open(QIODevice::ReadOnly), qPrintable(f.errorString()));
++ bytes = f.readAll();
++ }
++ QVERIFY(!bytes.isEmpty());
++
++ if (expectCRLF) {
++ QVERIFY2(bytes.contains("\r\n"), "Output should contain CRLF");
++ for (qsizetype i = 0; i < bytes.size(); ++i) {
++ if (bytes[i] == '\n')
++ QVERIFY2(i > 0 && bytes[i - 1] == '\r', "Bare LF found in CRLF output");
++ }
++ } else {
++ QVERIFY2(!bytes.contains("\r\n"), "Output should not contain CRLF");
++ }
++ }
++
++ QTEST_APPLESS_MAIN(tst_lconvert)
++
++ #include "tst_lconvert.moc"