qcollator_icu.cpp - mozsearch

++ // Copyright (C) 2020 The Qt Company Ltd.

++ // Copyright (C) 2013 Aleix Pol Gonzalez <aleixpol@kde.org>

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++ // Qt-Security score:critical reason:data-parser

++

++ #include "qcollator_p.h"

++ #include "qlocale_p.h"

++ #include "qstringlist.h"

++ #include "qstring.h"

++

++ #include <unicode/utypes.h>

++ #include <unicode/ucol.h>

++ #include <unicode/ustring.h>

++ #if QT_CONFIG(icu)

++ #include <unicode/ures.h>

++ #endif

++

++

++ #include "qdebug.h"

++ #include "qlibrary.h"

++

++ QT_BEGIN_NAMESPACE

++

++ // There are two cases in here, using the "real" ICU (ie. QT_CONFIG(icu)) and

++ // using Android's flavor of ICU provided on the system (slightly confusingly !QT_CONFIG(icu)).

++ // The Android case itself consists of two variants as well, new enough versions

++ // where we can dlopen the native ICU, and a JNI fallback.

++

++ #if !QT_CONFIG(icu) && defined(Q_OS_ANDROID)

++ using ucol_open_t = UCollator* (*)(const char*, UErrorCode*);

++ using ucol_setAttribute_t = void (*)(UCollator *, UColAttribute, UColAttributeValue, UErrorCode*);

++ using ucol_close_t = void (*)(UCollator*);

++ using ucol_strcoll_t = UCollationResult (*)(const UCollator*, const UChar*, int32_t, const UChar*, int32_t);

++ using ucol_getSortKey_t = int32_t (*)(const UCollator*, const UChar*, int32_t, uint8_t*, int32_t);

++ using u_errorName_t = const char* (*)(UErrorCode);

++

++ struct {

++     ucol_open_t open = nullptr;

++     ucol_setAttribute_t setAttribute = nullptr;

++     ucol_close_t close = nullptr;

++     ucol_strcoll_t strcoll = nullptr;

++     ucol_getSortKey_t getSortKey = nullptr;

++     u_errorName_t errorName = nullptr;

++ } static s_ucol;

++

++ #define ucol_open s_ucol.open

++ #define ucol_setAttribute s_ucol.setAttribute

++ #define ucol_close s_ucol.close

++ #define ucol_strcoll s_ucol.strcoll

++ #define ucol_getSortKey s_ucol.getSortKey

++ #define u_errorName s_ucol.errorName

++ #endif

++

++ void QCollatorPrivate::init()

++ {

++     cleanup();

++     if (isC())

++         return;

++

++ #if !QT_CONFIG(icu) && defined(Q_OS_ANDROID)

++     static bool icuLoaded = []() {

++         // available on Android API 33 or higher only

++         QLibrary icuLib(QStringLiteral("libicu"));

++         if (!icuLib.load()) {

++             qWarning().nospace() << "ICU loading failed: " << icuLib.errorString()

++                 << ". Fallback collator will be used instead, not all features will be available.";

++             return false;

++         }

++

++         s_ucol.open = reinterpret_cast<ucol_open_t>(icuLib.resolve("ucol_open"));

++         s_ucol.setAttribute = reinterpret_cast<ucol_setAttribute_t>(icuLib.resolve("ucol_setAttribute"));

++         s_ucol.close = reinterpret_cast<ucol_close_t>(icuLib.resolve("ucol_close"));

++         s_ucol.strcoll = reinterpret_cast<ucol_strcoll_t>(icuLib.resolve("ucol_strcoll"));

++         s_ucol.getSortKey = reinterpret_cast<ucol_getSortKey_t>(icuLib.resolve("ucol_getSortKey"));

++         s_ucol.errorName = reinterpret_cast<u_errorName_t>(icuLib.resolve("u_errorName"));

++         return s_ucol.open && s_ucol.setAttribute && s_ucol.close && s_ucol.strcoll && s_ucol.getSortKey;

++     }();

++

++     if (!icuLoaded) {

++         // on older Android versions, fall back to ICU Java API

++         // that works but has more overhead and offers less control

++         int strength = -1;

++         if (options.testFlag(Opt::DiacriticInsensitive) && options.testFlag(Opt::CaseInsensitive))

++             strength = UCOL_PRIMARY;

++         else if (options.testFlag(Opt::CaseInsensitive))

++             strength = UCOL_SECONDARY;

++         fallbackCollator = QtJniTypes::QtCollator::callStaticMethod<QtJniTypes::Collator>("getCollator", locale.bcp47Name(), strength);

++         dirty = false;

++         return;

++     }

++ #endif

++

++     auto errorName = [](UErrorCode status) -> const char * {

++ #if !QT_CONFIG(icu) && defined(Q_OS_ANDROID)

++         if (!s_ucol.errorName)

++             return "[Unknown ICU Error]";

++ #endif

++         const char *name = u_errorName(status);

++         // Always actually non-null (ICU v78.3) but not documented as such, so make sure:

++         return name ? name : "[Unknown ICU Error]";

++     };

++

++     UErrorCode status = U_ZERO_ERROR;

++     QByteArray name = QLocalePrivate::get(locale)->bcp47Name('_');

++     collator = ucol_open(name.constData(), &status);

++     if (U_FAILURE(status)) {

++         if (status == U_FILE_ACCESS_ERROR)

++             qFatal("QCollator: ICU data file is not accessible: %s", errorName(status));

++         qCWarning(lcQCollator, "Could not create collator: %s", errorName(status));

++         collator = nullptr;

++         dirty = false;

++         return;

++     }

++

++     // enable normalization by default

++     ucol_setAttribute(collator, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);

++

++     // The strength attribute in ICU is rather badly documented. Basically UCOL_PRIMARY

++     // ignores differences between base characters and accented characters as well as case.

++     // So A and A-umlaut would compare equal.

++     // UCOL_SECONDARY ignores case differences. UCOL_TERTIARY is the default in most languages

++     // and does case sensitive comparison.

++     // UCOL_QUATERNARY is used as default in a few languages such as Japanese to take care of some

++     // additional differences in those languages.

++     if (options.testFlag(Opt::DiacriticInsensitive)) {

++         // UCOL_PRIMARY ignores both diacritics and case

++         status = U_ZERO_ERROR;

++         ucol_setAttribute(collator, UCOL_STRENGTH, UCOL_PRIMARY, &status);

++         if (U_FAILURE(status)) {

++             qCWarning(lcQCollator,

++                       "ucol_setAttribute: Diacritic and case insensitivity failed: %s",

++                       errorName(status));

++         }

++

++         if (!options.testFlag(Opt::CaseInsensitive)) {

++             // Re-add case distinction if CaseInsensitive hasn't been set

++             status = U_ZERO_ERROR;

++             ucol_setAttribute(collator, UCOL_CASE_LEVEL, UCOL_ON, &status);

++             if (U_FAILURE(status)) {

++                 qCWarning(lcQCollator,

++                           "ucol_setAttribute: Diacritic insensitivity with case distinction failed:"

++                           " %s", errorName(status));

++             }

++         }

++     } else {

++         const UColAttributeValue strength

++                 = options.testFlag(Opt::CaseInsensitive) ? UCOL_SECONDARY : UCOL_DEFAULT_STRENGTH;

++         // Case sensitivity setting only

++         status = U_ZERO_ERROR;

++         ucol_setAttribute(collator, UCOL_STRENGTH, strength, &status);

++         if (U_FAILURE(status)) {

++             qCWarning(lcQCollator, "ucol_setAttribute: Case sensitivity failed: %s",

++                       errorName(status));

++         }

++     }

++

++     status = U_ZERO_ERROR;

++     ucol_setAttribute(collator, UCOL_NUMERIC_COLLATION,

++                       options.testFlag(Opt::NumericSort) ? UCOL_ON : UCOL_OFF, &status);

++     if (U_FAILURE(status)) {

++         qCWarning(lcQCollator, "ucol_setAttribute: numeric collation failed: %s",

++                   errorName(status));

++     }

++

++     status = U_ZERO_ERROR;

++     ucol_setAttribute(collator, UCOL_ALTERNATE_HANDLING,

++                       options.testFlag(Opt::IgnorePunctuation) ? UCOL_SHIFTED

++                                                                : UCOL_NON_IGNORABLE, &status);

++     if (U_FAILURE(status)) {

++         qCWarning(lcQCollator, "ucol_setAttribute: Alternate handling failed: %s",

++                   errorName(status));

++     }

++

++     dirty = false;

++ }

++

++ void QCollatorPrivate::cleanup()

++ {

++     if (collator)

++         ucol_close(collator);

++     collator = nullptr;

++ }

++

++ int QCollator::compare(QStringView s1, QStringView s2) const

++ {

++     if (!s1.size())

++         return s2.size() ? -1 : 0;

++     if (!s2.size())

++         return +1;

++

++     if (!d)

++         d = new QCollatorPrivate(QLocale().collation());

++

++     d->ensureInitialized();

++

++     if (d->collator) {

++         // truncating sizes (QTBUG-105038)

++         return ucol_strcoll(d->collator,

++                             reinterpret_cast<const UChar *>(s1.data()), s1.size(),

++                             reinterpret_cast<const UChar *>(s2.data()), s2.size());

++     }

++ #if !QT_CONFIG(icu) && defined(Q_OS_ANDROID)

++     else if (d->fallbackCollator.isValid()) {

++         return d->fallbackCollator.callMethod<int>("compare", s1, s2);

++     }

++ #endif

++

++     return QtPrivate::compareStrings(s1, s2, caseSensitivity());

++ }

++

++ QCollatorSortKey QCollator::sortKey(const QString &string) const

++ {

++     if (!d)

++         d = new QCollatorPrivate(QLocale().collation());

++

++     d->ensureInitialized();

++

++     if (d->isC())

++         return QCollatorPrivate::sortKeyFromData(string.toUtf8());

++

++     if (d->collator) {

++         QByteArray result(16 + string.size() + (string.size() >> 2), Qt::Uninitialized);

++         // truncating sizes (QTBUG-105038)

++         int size = ucol_getSortKey(d->collator, (const UChar *)string.constData(),

++                                    string.size(), (uint8_t *)result.data(), result.size());

++         if (size > result.size()) {

++             result.resize(size);

++             size = ucol_getSortKey(d->collator, (const UChar *)string.constData(),

++                                    string.size(), (uint8_t *)result.data(), result.size());

++         }

++         result.truncate(size);

++         return QCollatorPrivate::sortKeyFromData(std::move(result));

++     }

++ #if !QT_CONFIG(icu) && defined(Q_OS_ANDROID)

++     else if (d->fallbackCollator.isValid()) {

++         return QCollatorPrivate::sortKeyFromData(QtJniTypes::QtCollator::callStaticMethod<QByteArray>("getCollationKey", d->fallbackCollator, string));

++     }

++ #endif

++

++     return QCollatorPrivate::sortKeyFromData(QByteArray());

++ }

++

++ int QCollatorSortKey::compare(const QCollatorSortKey &otherKey) const noexcept

++ {

++     return d->m_key.compare(otherKey.d->m_key);

++ }

++

++ QT_END_NAMESPACE

Revision control