qhelpsearchindexwriter.cpp

++ // Copyright (C) 2016 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++ // Qt-Security score:critical reason:data-parser

++

++ #include "qhelpsearchindexwriter_p.h"

++ #include "qhelp_global.h"

++ #include "qhelpdbreader_p.h"

++ #include "qhelpenginecore.h"

++

++ #include <QtTools/qttools-config.h>

++ #include <QtCore/qdatastream.h>

++ #include <QtCore/qdatetime.h>

++ #include <QtCore/qdir.h>

++ #include <QtCore/qstringconverter.h>

++ #include <QtCore/qtextstream.h>

++ #include <QtCore/qurl.h>

++ #include <QtCore/qvariant.h>

++ #if QT_CONFIG(fullqthelp)

++ #  include <QtGui/qtextdocument.h>

++ #endif

++ #include <QtSql/qsqldatabase.h>

++ #include <QtSql/qsqldriver.h>

++ #include <QtSql/qsqlerror.h>

++ #include <QtSql/qsqlquery.h>

++

++ QT_BEGIN_NAMESPACE

++

++ using namespace Qt::StringLiterals;

++

++ namespace fulltextsearch {

++

++ const char FTS_DB_NAME[] = "fts";

++

++ class Writer

++ {

++ public:

++     Writer(const QString &path);

++     ~Writer();

++

++     bool tryInit(bool reindex);

++     void flush();

++

++     void removeNamespace(const QString &namespaceName);

++     bool hasNamespace(const QString &namespaceName);

++     void insertDoc(const QString &namespaceName,

++                    const QString &attributes,

++                    const QString &url,

++                    const QString &title,

++                    const QString &contents);

++     void startTransaction();

++     void endTransaction();

++

++ private:

++     void init(bool reindex);

++     bool hasDB();

++

++     const QString m_dbDir;

++     QString m_uniqueId;

++

++     bool m_needOptimize = false;

++     QSqlDatabase m_db;

++     QVariantList m_namespaces;

++     QVariantList m_attributes;

++     QVariantList m_urls;

++     QVariantList m_titles;

++     QVariantList m_contents;

++ };

++

++ Writer::Writer(const QString &path)

++     : m_dbDir(path)

++ {

++     QDir().mkpath(m_dbDir);

++     m_uniqueId = QHelpGlobal::uniquifyConnectionName("QHelpWriter"_L1, this);

++     m_db = QSqlDatabase::addDatabase("QSQLITE"_L1, m_uniqueId);

++     const QString dbPath = m_dbDir + u'/' + QLatin1StringView(FTS_DB_NAME);

++     m_db.setDatabaseName(dbPath);

++     if (!m_db.open()) {

++         const QString &error = QHelpSearchIndexWriter::tr(

++                                        "Cannot open database \"%1\" using connection \"%2\": %3")

++                                        .arg(dbPath, m_uniqueId, m_db.lastError().text());

++         qWarning("%s", qUtf8Printable(error));

++         m_db = {};

++         QSqlDatabase::removeDatabase(m_uniqueId);

++         m_uniqueId.clear();

++     } else {

++         startTransaction();

++     }

++ }

++

++ bool Writer::tryInit(bool reindex)

++ {

++     if (!m_db.isValid())

++         return true;

++

++     QSqlQuery query(m_db);

++     // HACK: we try to perform any modifying command just to check if

++     // we don't get SQLITE_BUSY code (SQLITE_BUSY is defined to 5 in sqlite driver)

++     if (!query.exec("CREATE TABLE foo ();"_L1) && query.lastError().nativeErrorCode() == "5"_L1) // db is locked

++         return false;

++

++     // HACK: clear what we have created

++     query.exec("DROP TABLE foo;"_L1);

++

++     init(reindex);

++     return true;

++ }

++

++ bool Writer::hasDB()

++ {

++     if (!m_db.isValid())

++         return false;

++

++     QSqlQuery query(m_db);

++     query.prepare("SELECT id FROM info LIMIT 1"_L1);

++     query.exec();

++     return query.next();

++ }

++

++ void Writer::init(bool reindex)

++ {

++     if (!m_db.isValid())

++         return;

++

++     QSqlQuery query(m_db);

++

++     if (reindex && hasDB()) {

++         m_needOptimize = true;

++

++         query.exec("DROP TABLE titles;"_L1);

++         query.exec("DROP TABLE contents;"_L1);

++         query.exec("DROP TABLE info;"_L1);

++     }

++

++     query.exec("CREATE TABLE info (id INTEGER PRIMARY KEY, namespace, attributes, url, title, data);"_L1);

++

++     query.exec("CREATE VIRTUAL TABLE titles USING fts5("

++                "namespace UNINDEXED, attributes UNINDEXED, "

++                "url UNINDEXED, title, "

++                "tokenize = 'porter unicode61', content = 'info', content_rowid='id');"_L1);

++     query.exec("CREATE TRIGGER titles_insert AFTER INSERT ON info BEGIN "

++                "INSERT INTO titles(rowid, namespace, attributes, url, title) "

++                "VALUES(new.id, new.namespace, new.attributes, new.url, new.title); "

++                "END;"_L1);

++     query.exec("CREATE TRIGGER titles_delete AFTER DELETE ON info BEGIN "

++                "INSERT INTO titles(titles, rowid, namespace, attributes, url, title) "

++                "VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title); "

++                "END;"_L1);

++     query.exec("CREATE TRIGGER titles_update AFTER UPDATE ON info BEGIN "

++                "INSERT INTO titles(titles, rowid, namespace, attributes, url, title) "

++                "VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title); "

++                "INSERT INTO titles(rowid, namespace, attributes, url, title) "

++                "VALUES(new.id, new.namespace, new.attributes, new.url, new.title); "

++                "END;"_L1);

++

++     query.exec("CREATE VIRTUAL TABLE contents USING fts5("

++                "namespace UNINDEXED, attributes UNINDEXED, "

++                "url UNINDEXED, title, data, "

++                "tokenize = 'porter unicode61', content = 'info', content_rowid='id');"_L1);

++     query.exec("CREATE TRIGGER contents_insert AFTER INSERT ON info BEGIN "

++                "INSERT INTO contents(rowid, namespace, attributes, url, title, data) "

++                "VALUES(new.id, new.namespace, new.attributes, new.url, new.title, new.data); "

++                "END;"_L1);

++     query.exec("CREATE TRIGGER contents_delete AFTER DELETE ON info BEGIN "

++                "INSERT INTO contents(contents, rowid, namespace, attributes, url, title, data) "

++                "VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title, old.data); "

++                "END;"_L1);

++     query.exec("CREATE TRIGGER contents_update AFTER UPDATE ON info BEGIN "

++                "INSERT INTO contents(contents, rowid, namespace, attributes, url, title, data) "

++                "VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title, old.data); "

++                "INSERT INTO contents(rowid, namespace, attributes, url, title, data) "

++                "VALUES(new.id, new.namespace, new.attributes, new.url, new.title, new.data); "

++                "END;"_L1);

++ }

++

++ Writer::~Writer()

++ {

++     if (m_db.isValid())

++         m_db.close();

++     m_db = {};

++     if (!m_uniqueId.isEmpty())

++         QSqlDatabase::removeDatabase(m_uniqueId);

++ }

++

++ void Writer::flush()

++ {

++     if (!m_db.isValid())

++         return;

++

++     QSqlQuery query(m_db);

++     query.prepare("INSERT INTO info (namespace, attributes, url, title, data) VALUES (?, ?, ?, ?, ?)"_L1);

++     query.addBindValue(m_namespaces);

++     query.addBindValue(m_attributes);

++     query.addBindValue(m_urls);

++     query.addBindValue(m_titles);

++     query.addBindValue(m_contents);

++     query.execBatch();

++

++     m_namespaces.clear();

++     m_attributes.clear();

++     m_urls.clear();

++     m_titles.clear();

++     m_contents.clear();

++ }

++

++ void Writer::removeNamespace(const QString &namespaceName)

++ {

++     if (!m_db.isValid() || !hasNamespace(namespaceName)) // no data to delete

++         return;

++

++     m_needOptimize = true;

++     QSqlQuery query(m_db);

++     query.prepare("DELETE FROM info WHERE namespace = ?"_L1);

++     query.addBindValue(namespaceName);

++     query.exec();

++ }

++

++ bool Writer::hasNamespace(const QString &namespaceName)

++ {

++     if (!m_db.isValid())

++         return false;

++

++     QSqlQuery query(m_db);

++     query.prepare("SELECT id FROM info WHERE namespace = ? LIMIT 1"_L1);

++     query.addBindValue(namespaceName);

++     query.exec();

++     return query.next();

++ }

++

++ void Writer::insertDoc(const QString &namespaceName,

++                        const QString &attributes,

++                        const QString &url,

++                        const QString &title,

++                        const QString &contents)

++ {

++     m_namespaces.append(namespaceName);

++     m_attributes.append(attributes);

++     m_urls.append(url);

++     m_titles.append(title);

++     m_contents.append(contents);

++ }

++

++ void Writer::startTransaction()

++ {

++     if (!m_db.isValid())

++         return;

++

++     m_needOptimize = false;

++     if (m_db.driver()->hasFeature(QSqlDriver::Transactions))

++         m_db.transaction();

++ }

++

++ void Writer::endTransaction()

++ {

++     if (!m_db.isValid())

++         return;

++

++     QSqlQuery query(m_db);

++

++     if (m_needOptimize) {

++         query.exec("INSERT INTO titles(titles) VALUES('rebuild')"_L1);

++         query.exec("INSERT INTO contents(contents) VALUES('rebuild')"_L1);

++     }

++

++     if (m_db.driver()->hasFeature(QSqlDriver::Transactions))

++         m_db.commit();

++

++     if (m_needOptimize)

++         query.exec("VACUUM"_L1);

++ }

++

++ QHelpSearchIndexWriter::~QHelpSearchIndexWriter()

++ {

++     m_mutex.lock();

++     this->m_cancel = true;

++     m_mutex.unlock();

++     wait();

++ }

++

++ void QHelpSearchIndexWriter::cancelIndexing()

++ {

++     QMutexLocker lock(&m_mutex);

++     m_cancel = true;

++ }

++

++ void QHelpSearchIndexWriter::updateIndex(const QString &collectionFile,

++                                          const QString &indexFilesFolder, bool reindex)

++ {

++     wait();

++     QMutexLocker lock(&m_mutex);

++

++     m_cancel = false;

++     m_reindex = reindex;

++     m_collectionFile = collectionFile;

++     m_indexFilesFolder = indexFilesFolder;

++

++     lock.unlock();

++

++     start(QThread::LowestPriority);

++ }

++

++ static const char IndexedNamespacesKey[] = "FTS5IndexedNamespaces";

++

++ static QMap<QString, QDateTime> readIndexMap(const QHelpEngineCore &engine)

++ {

++     QMap<QString, QDateTime> indexMap;

++     QDataStream dataStream(

++             engine.customValue(QLatin1StringView(IndexedNamespacesKey)).toByteArray());

++     dataStream >> indexMap;

++     return indexMap;

++ }

++

++ static bool writeIndexMap(QHelpEngineCore *engine, const QMap<QString, QDateTime> &indexMap)

++ {

++     QByteArray data;

++     QDataStream dataStream(&data, QIODevice::ReadWrite);

++     dataStream << indexMap;

++     return engine->setCustomValue(QLatin1StringView(IndexedNamespacesKey), data);

++ }

++

++ static bool clearIndexMap(QHelpEngineCore *engine)

++ {

++     return engine->removeCustomValue(QLatin1StringView(IndexedNamespacesKey));

++ }

++

++ void QHelpSearchIndexWriter::run()

++ {

++     QMutexLocker lock(&m_mutex);

++

++     if (m_cancel)

++         return;

++

++     const bool reindex(m_reindex);

++     const QString collectionFile(m_collectionFile);

++     const QString indexPath(m_indexFilesFolder);

++

++     lock.unlock();

++

++     QHelpEngineCore engine(collectionFile, nullptr);

++     if (!engine.setupData())

++         return;

++

++     if (reindex)

++         clearIndexMap(&engine);

++

++     emit indexingStarted();

++

++     Writer writer(indexPath);

++

++     while (!writer.tryInit(reindex)) {

++         lock.relock();

++         if (m_cancel) {

++             emit indexingFinished();

++             return;

++         }

++         lock.unlock();

++         sleep(1);

++     }

++

++     const QStringList &registeredDocs = engine.registeredDocumentations();

++     QMap<QString, QDateTime> indexMap = readIndexMap(engine);

++

++     if (!reindex) {

++         for (const QString &namespaceName : registeredDocs) {

++             const auto it = indexMap.constFind(namespaceName);

++             if (it != indexMap.constEnd()) {

++                 const QString path = engine.documentationFileName(namespaceName);

++                 if (*it < QFileInfo(path).lastModified()) {

++                     // Remove some outdated indexed stuff

++                     indexMap.erase(it);

++                     writer.removeNamespace(namespaceName);

++                 } else if (!writer.hasNamespace(namespaceName)) {

++                     // No data in fts db for namespace.

++                     // The namespace could have been removed from fts db

++                     // or the whole fts db have been removed

++                     // without removing it from indexMap.

++                     indexMap.erase(it);

++                 }

++             } else {

++                 // Needed in case namespaceName was removed from indexMap

++                 // without removing it from fts db.

++                 // May happen when e.g. qch file was removed manually

++                 // without removing fts db.

++                 writer.removeNamespace(namespaceName);

++             }

++         // TODO: we may also detect if there are any other data

++         // and remove it

++         }

++     } else {

++         indexMap.clear();

++     }

++

++     auto it = indexMap.begin();

++     while (it != indexMap.end()) {

++         if (!registeredDocs.contains(it.key())) {

++             writer.removeNamespace(it.key());

++             it = indexMap.erase(it);

++         } else {

++             ++it;

++         }

++     }

++

++     for (const QString &namespaceName : registeredDocs) {

++         lock.relock();

++         if (m_cancel) {

++             // store what we have done so far

++             writeIndexMap(&engine, indexMap);

++             writer.endTransaction();

++             emit indexingFinished();

++             return;

++         }

++         lock.unlock();

++

++         // if indexed, continue

++         if (indexMap.contains(namespaceName))

++             continue;

++

++         const QString fileName = engine.documentationFileName(namespaceName);

++         QHelpDBReader reader(fileName, QHelpGlobal::uniquifyConnectionName(

++                                  fileName, this), nullptr);

++         if (!reader.init())

++             continue;

++

++         const QString virtualFolder = reader.virtualFolder();

++

++         const QList<QStringList> &attributeSets =

++             engine.filterAttributeSets(namespaceName);

++

++         for (const QStringList &attributes : attributeSets) {

++             const QString &attributesString = attributes.join(u'|');

++

++             bool canceled = false;

++             reader.filesData(attributes, {"html"_L1, "htm"_L1, "txt"_L1},

++                              [&](const QString &file, const QByteArray &data) {

++                 lock.relock();

++                 if (m_cancel) {

++                     canceled = true;

++                     return false;

++                 }

++                 lock.unlock();

++

++                 if (data.isEmpty())

++                     return true;

++

++                 QUrl url;

++                 url.setScheme("qthelp"_L1);

++                 url.setAuthority(namespaceName);

++                 url.setPath(u'/' + virtualFolder + u'/' + file);

++

++                 if (url.hasFragment())

++                     url.setFragment({});

++

++                 const QString &fullFileName = url.toString();

++                 if (!fullFileName.endsWith(".html"_L1) && !fullFileName.endsWith(".htm"_L1)

++                     && !fullFileName.endsWith(".txt"_L1)) {

++                     return true;

++                 }

++

++                 QTextStream s(data);

++                 auto encoding = QStringDecoder::encodingForHtml(data);

++                 if (encoding)

++                     s.setEncoding(*encoding);

++

++                 const QString &text = s.readAll();

++                 if (text.isEmpty())

++                     return true;

++

++                 QString title;

++                 QString contents;

++                 if (fullFileName.endsWith(".txt"_L1)) {

++                     title = fullFileName.mid(fullFileName.lastIndexOf(u'/') + 1);

++                     contents = text.toHtmlEscaped();

++                 } else {

++ #if QT_CONFIG(fullqthelp) && QT_CONFIG(texthtmlparser)

++                     QTextDocument doc;

++                     doc.setHtml(text);

++

++                     title = doc.metaInformation(QTextDocument::DocumentTitle).toHtmlEscaped();

++                     contents = doc.toPlainText().toHtmlEscaped();

++ #else

++                     return;

++ #endif

++                 }

++

++                 writer.insertDoc(namespaceName, attributesString, fullFileName, title, contents);

++                 return true;

++             });

++

++             if (canceled) {

++                 // store what we have done so far

++                 writeIndexMap(&engine, indexMap);

++                 writer.endTransaction();

++                 emit indexingFinished();

++                 return;

++             }

++         }

++         writer.flush();

++         const QString &path = engine.documentationFileName(namespaceName);

++         indexMap.insert(namespaceName, QFileInfo(path).lastModified());

++     }

++

++     writeIndexMap(&engine, indexMap);

++

++     writer.endTransaction();

++     emit indexingFinished();

++ }

++

++ } // namespace fulltextsearch

++

++ QT_END_NAMESPACE

Revision control