Revision control
++ // Copyright (C) 2018 The Qt Company Ltd.
++ // Copyright (C) 2018 Intel Corporation.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
++
++ #ifndef UTF8_DATA_H
++ #define UTF8_DATA_H
++
++ #include <QtTest/QtTest>
++
++ QT_BEGIN_NAMESPACE
++
++ void loadInvalidUtf8Rows()
++ {
++ // Wrong continuations
++ QTest::newRow("bad-continuation-1char") << QByteArray("\x80");
++ QTest::newRow("bad-continuation-2chars-1") << QByteArray("\xC2\xC0");
++ QTest::newRow("bad-continuation-2chars-2") << QByteArray("\xC3\xDF");
++ QTest::newRow("bad-continuation-2chars-3") << QByteArray("\xC7\xF0");
++ QTest::newRow("bad-continuation-3chars-1") << QByteArray("\xE0\xA0\xC0");
++ QTest::newRow("bad-continuation-3chars-2") << QByteArray("\xE0\xC0\xA0");
++ QTest::newRow("bad-continuation-4chars-1") << QByteArray("\xF0\x90\x80\xC0");
++ QTest::newRow("bad-continuation-4chars-2") << QByteArray("\xF0\x90\xC0\x80");
++ QTest::newRow("bad-continuation-4chars-3") << QByteArray("\xF0\xC0\x80\x80");
++
++ // Too short
++ QTest::newRow("too-short-2chars") << QByteArray("\xC2");
++ QTest::newRow("too-short-3chars-1") << QByteArray("\xE0");
++ QTest::newRow("too-short-3chars-2") << QByteArray("\xE0\xA0");
++ QTest::newRow("too-short-4chars-1") << QByteArray("\xF0");
++ QTest::newRow("too-short-4chars-2") << QByteArray("\xF0\x90");
++ QTest::newRow("too-short-4chars-3") << QByteArray("\xF0\x90\x80");
++
++ // Surrogate pairs must now be present either
++ // U+D800: 1101 10 0000 00 0000
++ // encoding: xxxz:1101 xz10:0000 xz00:0000
++ QTest::newRow("hi-surrogate") << QByteArray("\xED\xA0\x80");
++ // U+DC00: 1101 11 0000 00 0000
++ // encoding: xxxz:1101 xz11:0000 xz00:0000
++ QTest::newRow("lo-surrogate") << QByteArray("\xED\xB0\x80");
++
++ // not even in pair:
++ QTest::newRow("surrogate-pair") << QByteArray("\xED\xA0\x80\xED\xB0\x80");
++
++ // Characters outside the Unicode range:
++ // 0x110000: 00 0100 01 0000 00 0000 00 0000
++ // encoding: xxxx:z100 xz01:0000 xz00:0000 xz00:0000
++ QTest::newRow("non-unicode-1") << QByteArray("\xF4\x90\x80\x80");
++ // 0x200000: 00 1000 00 0000 00 0000 00 0000
++ // encoding: xxxx:xz00 xz00:1000 xz00:0000 xz00:0000 xz00:0000
++ QTest::newRow("non-unicode-2") << QByteArray("\xF8\x88\x80\x80\x80");
++ // 0x04000000: 0100 00 0000 00 0000 00 0000 00 0000
++ // encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001
++ QTest::newRow("non-unicode-3") << QByteArray("\xFC\x84\x80\x80\x80\x80");
++ // 0x7fffffff: 1 11 1111 11 1111 11 1111 11 1111 11 1111
++ // encoding: xxxx:xxz0 xz00:0100 xz00:0000 xz00:0000 xz00:0001 xz00:0001
++ QTest::newRow("non-unicode-4") << QByteArray("\xFD\xBF\xBF\xBF\xBF\xBF");
++
++ // As seen above, 0xFE and 0xFF never appear:
++ QTest::newRow("fe") << QByteArray("\xFE");
++ QTest::newRow("fe-bis") << QByteArray("\xFE\xBF\xBF\xBF\xBF\xBF\xBF");
++ QTest::newRow("ff") << QByteArray("\xFF");
++ QTest::newRow("ff-bis") << QByteArray("\xFF\xBF\xBF\xBF\xBF\xBF\xBF\xBF");
++
++ // some combinations in UTF-8 are invalid even though they have the proper bits set
++ // these are known as overlong sequences
++
++ // "A": U+0041: 01 00 0001
++ // overlong 2: xxz0:0001 xz00:0001
++ QTest::newRow("overlong-1-2") << QByteArray("\xC1\x81");
++ // overlong 3: xxxz:0000 xz00:0001 xz00:0001
++ QTest::newRow("overlong-1-3") << QByteArray("\xE0\x81\x81");
++ // overlong 4: xxxx:z000 xz00:0000 xz00:0001 xz00:0001
++ QTest::newRow("overlong-1-4") << QByteArray("\xF0\x80\x81\x81");
++ // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0001 xz00:0001
++ QTest::newRow("overlong-1-5") << QByteArray("\xF8\x80\x80\x81\x81");
++ // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0001 xz00:0001
++ QTest::newRow("overlong-1-6") << QByteArray("\xFC\x80\x80\x80\x81\x81");
++
++ // U+0080: 10 00 0000
++ // proper encoding: xxz0:0010 xz00:0000
++ // overlong 3: xxxz:0000 xz00:0010 xz00:0000
++ QTest::newRow("overlong-2-3") << QByteArray("\xE0\x82\x80");
++ // overlong 4: xxxx:z000 xz00:0000 xz00:0010 xz00:0000
++ QTest::newRow("overlong-2-4") << QByteArray("\xF0\x80\x82\x80");
++ // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz00:0010 xz00:0000
++ QTest::newRow("overlong-2-5") << QByteArray("\xF8\x80\x80\x82\x80");
++ // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz00:0010 xz00:0000
++ QTest::newRow("overlong-2-6") << QByteArray("\xFC\x80\x80\x80\x82\x80");
++
++ // U+0800: 10 0000 00 0000
++ // proper encoding: xxxz:0000 xz10:0000 xz00:0000
++ // overlong 4: xxxx:z000 xz00:0000 xz10:0000 xz00:0000
++ QTest::newRow("overlong-3-4") << QByteArray("\xF0\x80\xA0\x80");
++ // overlong 5: xxxx:xz00 xz00:0000 xz00:0000 xz10:0000 xz00:0000
++ QTest::newRow("overlong-3-5") << QByteArray("\xF8\x80\x80\xA0\x80");
++ // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0000 xz10:0000 xz00:0000
++ QTest::newRow("overlong-3-6") << QByteArray("\xFC\x80\x80\x80\xA0\x80");
++
++ // U+010000: 00 0100 00 0000 00 0000
++ // proper encoding: xxxx:z000 xz00:0100 xz00:0000 xz00:0000
++ // overlong 5: xxxx:xz00 xz00:0000 xz00:0100 xz00:0000 xz00:0000
++ QTest::newRow("overlong-4-5") << QByteArray("\xF8\x80\x84\x80\x80");
++ // overlong 6: xxxx:xxz0 xz00:0000 xz00:0000 xz00:0100 xz00:0000 xz00:0000
++ QTest::newRow("overlong-4-6") << QByteArray("\xFC\x80\x80\x84\x80\x80");
++
++ }
++
++ void loadNonCharactersRows()
++ {
++ // Unicode has a couple of "non-characters" that one can use internally
++ // These characters are allowed for text-interchange (see http://www.unicode.org/versions/corrigendum9.html)
++ //
++ // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF,
++ // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and
++ // U+FDEF (inclusive)
++
++ // U+FDD0 through U+FDEF
++ for (int i = 0; i < 16; ++i) {
++ char utf8[] = { char(0357), char(0267), char(0220 + i), 0 };
++ QString utf16 = QChar(0xfdd0 + i);
++ QTest::newRow(qPrintable(QString::number(0xfdd0 + i, 16))) << QByteArray(utf8) << utf16;
++ }
++
++ // the last two in Planes 1 through 16
++ for (uint plane = 1; plane <= 16; ++plane) {
++ for (uint lower = 0xfffe; lower < 0x10000; ++lower) {
++ uint ucs4 = (plane << 16) | lower;
++ char utf8[] = { char(0xf0 | uchar(ucs4 >> 18)),
++ char(0x80 | (uchar(ucs4 >> 12) & 0x3f)),
++ char(0x80 | (uchar(ucs4 >> 6) & 0x3f)),
++ char(0x80 | (uchar(ucs4) & 0x3f)),
++ 0 };
++ const auto utf16 = QChar::fromUcs4(ucs4);
++
++ QTest::newRow(qPrintable(QString::number(ucs4, 16))) << QByteArray(utf8) << QStringView{utf16}.toString();
++ }
++ }
++
++ QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE") << QString(QChar(0xfffe));
++ QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF") << QString(QChar(0xffff));
++ }
++
++ QT_END_NAMESPACE
++
++ #endif