Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
// Qt-Security score:critical reason:data-parser
#include "qhelpsearchindexwriter_p.h"
#include "qhelp_global.h"
#include "qhelpdbreader_p.h"
#include "qhelpenginecore.h"
#include <QtTools/qttools-config.h>
#include <QtCore/qdatastream.h>
#include <QtCore/qdatetime.h>
#include <QtCore/qdir.h>
#include <QtCore/qstringconverter.h>
#include <QtCore/qtextstream.h>
#include <QtCore/qurl.h>
#include <QtCore/qvariant.h>
#if QT_CONFIG(fullqthelp)
# include <QtGui/qtextdocument.h>
#endif
#include <QtSql/qsqldatabase.h>
#include <QtSql/qsqldriver.h>
#include <QtSql/qsqlerror.h>
#include <QtSql/qsqlquery.h>
QT_BEGIN_NAMESPACE
using namespace Qt::StringLiterals;
namespace fulltextsearch {
const char FTS_DB_NAME[] = "fts";
class Writer
{
public:
Writer(const QString &path);
~Writer();
bool tryInit(bool reindex);
void flush();
void removeNamespace(const QString &namespaceName);
bool hasNamespace(const QString &namespaceName);
void insertDoc(const QString &namespaceName,
const QString &attributes,
const QString &url,
const QString &title,
const QString &contents);
void startTransaction();
void endTransaction();
private:
void init(bool reindex);
bool hasDB();
const QString m_dbDir;
QString m_uniqueId;
bool m_needOptimize = false;
QSqlDatabase m_db;
QVariantList m_namespaces;
QVariantList m_attributes;
QVariantList m_urls;
QVariantList m_titles;
QVariantList m_contents;
};
Writer::Writer(const QString &path)
: m_dbDir(path)
{
QDir().mkpath(m_dbDir);
m_uniqueId = QHelpGlobal::uniquifyConnectionName("QHelpWriter"_L1, this);
m_db = QSqlDatabase::addDatabase("QSQLITE"_L1, m_uniqueId);
const QString dbPath = m_dbDir + u'/' + QLatin1StringView(FTS_DB_NAME);
m_db.setDatabaseName(dbPath);
if (!m_db.open()) {
const QString &error = QHelpSearchIndexWriter::tr(
"Cannot open database \"%1\" using connection \"%2\": %3")
.arg(dbPath, m_uniqueId, m_db.lastError().text());
qWarning("%s", qUtf8Printable(error));
m_db = {};
QSqlDatabase::removeDatabase(m_uniqueId);
m_uniqueId.clear();
} else {
startTransaction();
}
}
bool Writer::tryInit(bool reindex)
{
if (!m_db.isValid())
return true;
QSqlQuery query(m_db);
// HACK: we try to perform any modifying command just to check if
// we don't get SQLITE_BUSY code (SQLITE_BUSY is defined to 5 in sqlite driver)
if (!query.exec("CREATE TABLE foo ();"_L1) && query.lastError().nativeErrorCode() == "5"_L1) // db is locked
return false;
// HACK: clear what we have created
query.exec("DROP TABLE foo;"_L1);
init(reindex);
return true;
}
bool Writer::hasDB()
{
if (!m_db.isValid())
return false;
QSqlQuery query(m_db);
query.prepare("SELECT id FROM info LIMIT 1"_L1);
query.exec();
return query.next();
}
void Writer::init(bool reindex)
{
if (!m_db.isValid())
return;
QSqlQuery query(m_db);
if (reindex && hasDB()) {
m_needOptimize = true;
query.exec("DROP TABLE titles;"_L1);
query.exec("DROP TABLE contents;"_L1);
query.exec("DROP TABLE info;"_L1);
}
query.exec("CREATE TABLE info (id INTEGER PRIMARY KEY, namespace, attributes, url, title, data);"_L1);
query.exec("CREATE VIRTUAL TABLE titles USING fts5("
"namespace UNINDEXED, attributes UNINDEXED, "
"url UNINDEXED, title, "
"tokenize = 'porter unicode61', content = 'info', content_rowid='id');"_L1);
query.exec("CREATE TRIGGER titles_insert AFTER INSERT ON info BEGIN "
"INSERT INTO titles(rowid, namespace, attributes, url, title) "
"VALUES(new.id, new.namespace, new.attributes, new.url, new.title); "
"END;"_L1);
query.exec("CREATE TRIGGER titles_delete AFTER DELETE ON info BEGIN "
"INSERT INTO titles(titles, rowid, namespace, attributes, url, title) "
"VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title); "
"END;"_L1);
query.exec("CREATE TRIGGER titles_update AFTER UPDATE ON info BEGIN "
"INSERT INTO titles(titles, rowid, namespace, attributes, url, title) "
"VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title); "
"INSERT INTO titles(rowid, namespace, attributes, url, title) "
"VALUES(new.id, new.namespace, new.attributes, new.url, new.title); "
"END;"_L1);
query.exec("CREATE VIRTUAL TABLE contents USING fts5("
"namespace UNINDEXED, attributes UNINDEXED, "
"url UNINDEXED, title, data, "
"tokenize = 'porter unicode61', content = 'info', content_rowid='id');"_L1);
query.exec("CREATE TRIGGER contents_insert AFTER INSERT ON info BEGIN "
"INSERT INTO contents(rowid, namespace, attributes, url, title, data) "
"VALUES(new.id, new.namespace, new.attributes, new.url, new.title, new.data); "
"END;"_L1);
query.exec("CREATE TRIGGER contents_delete AFTER DELETE ON info BEGIN "
"INSERT INTO contents(contents, rowid, namespace, attributes, url, title, data) "
"VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title, old.data); "
"END;"_L1);
query.exec("CREATE TRIGGER contents_update AFTER UPDATE ON info BEGIN "
"INSERT INTO contents(contents, rowid, namespace, attributes, url, title, data) "
"VALUES('delete', old.id, old.namespace, old.attributes, old.url, old.title, old.data); "
"INSERT INTO contents(rowid, namespace, attributes, url, title, data) "
"VALUES(new.id, new.namespace, new.attributes, new.url, new.title, new.data); "
"END;"_L1);
}
Writer::~Writer()
{
if (m_db.isValid())
m_db.close();
m_db = {};
if (!m_uniqueId.isEmpty())
QSqlDatabase::removeDatabase(m_uniqueId);
}
void Writer::flush()
{
if (!m_db.isValid())
return;
QSqlQuery query(m_db);
query.prepare("INSERT INTO info (namespace, attributes, url, title, data) VALUES (?, ?, ?, ?, ?)"_L1);
query.addBindValue(m_namespaces);
query.addBindValue(m_attributes);
query.addBindValue(m_urls);
query.addBindValue(m_titles);
query.addBindValue(m_contents);
query.execBatch();
m_namespaces.clear();
m_attributes.clear();
m_urls.clear();
m_titles.clear();
m_contents.clear();
}
void Writer::removeNamespace(const QString &namespaceName)
{
if (!m_db.isValid() || !hasNamespace(namespaceName)) // no data to delete
return;
m_needOptimize = true;
QSqlQuery query(m_db);
query.prepare("DELETE FROM info WHERE namespace = ?"_L1);
query.addBindValue(namespaceName);
query.exec();
}
bool Writer::hasNamespace(const QString &namespaceName)
{
if (!m_db.isValid())
return false;
QSqlQuery query(m_db);
query.prepare("SELECT id FROM info WHERE namespace = ? LIMIT 1"_L1);
query.addBindValue(namespaceName);
query.exec();
return query.next();
}
void Writer::insertDoc(const QString &namespaceName,
const QString &attributes,
const QString &url,
const QString &title,
const QString &contents)
{
m_namespaces.append(namespaceName);
m_attributes.append(attributes);
m_urls.append(url);
m_titles.append(title);
m_contents.append(contents);
}
void Writer::startTransaction()
{
if (!m_db.isValid())
return;
m_needOptimize = false;
if (m_db.driver()->hasFeature(QSqlDriver::Transactions))
m_db.transaction();
}
void Writer::endTransaction()
{
if (!m_db.isValid())
return;
QSqlQuery query(m_db);
if (m_needOptimize) {
query.exec("INSERT INTO titles(titles) VALUES('rebuild')"_L1);
query.exec("INSERT INTO contents(contents) VALUES('rebuild')"_L1);
}
if (m_db.driver()->hasFeature(QSqlDriver::Transactions))
m_db.commit();
if (m_needOptimize)
query.exec("VACUUM"_L1);
}
QHelpSearchIndexWriter::~QHelpSearchIndexWriter()
{
m_mutex.lock();
this->m_cancel = true;
m_mutex.unlock();
wait();
}
void QHelpSearchIndexWriter::cancelIndexing()
{
QMutexLocker lock(&m_mutex);
m_cancel = true;
}
void QHelpSearchIndexWriter::updateIndex(const QString &collectionFile,
const QString &indexFilesFolder, bool reindex)
{
wait();
QMutexLocker lock(&m_mutex);
m_cancel = false;
m_reindex = reindex;
m_collectionFile = collectionFile;
m_indexFilesFolder = indexFilesFolder;
lock.unlock();
start(QThread::LowestPriority);
}
static const char IndexedNamespacesKey[] = "FTS5IndexedNamespaces";
static QMap<QString, QDateTime> readIndexMap(const QHelpEngineCore &engine)
{
QMap<QString, QDateTime> indexMap;
QDataStream dataStream(
engine.customValue(QLatin1StringView(IndexedNamespacesKey)).toByteArray());
dataStream >> indexMap;
return indexMap;
}
static bool writeIndexMap(QHelpEngineCore *engine, const QMap<QString, QDateTime> &indexMap)
{
QByteArray data;
QDataStream dataStream(&data, QIODevice::ReadWrite);
dataStream << indexMap;
return engine->setCustomValue(QLatin1StringView(IndexedNamespacesKey), data);
}
static bool clearIndexMap(QHelpEngineCore *engine)
{
return engine->removeCustomValue(QLatin1StringView(IndexedNamespacesKey));
}
void QHelpSearchIndexWriter::run()
{
QMutexLocker lock(&m_mutex);
if (m_cancel)
return;
const bool reindex(m_reindex);
const QString collectionFile(m_collectionFile);
const QString indexPath(m_indexFilesFolder);
lock.unlock();
QHelpEngineCore engine(collectionFile, nullptr);
if (!engine.setupData())
return;
if (reindex)
clearIndexMap(&engine);
emit indexingStarted();
Writer writer(indexPath);
while (!writer.tryInit(reindex)) {
lock.relock();
if (m_cancel) {
emit indexingFinished();
return;
}
lock.unlock();
sleep(1);
}
const QStringList ®isteredDocs = engine.registeredDocumentations();
QMap<QString, QDateTime> indexMap = readIndexMap(engine);
if (!reindex) {
for (const QString &namespaceName : registeredDocs) {
const auto it = indexMap.constFind(namespaceName);
if (it != indexMap.constEnd()) {
const QString path = engine.documentationFileName(namespaceName);
if (*it < QFileInfo(path).lastModified()) {
// Remove some outdated indexed stuff
indexMap.erase(it);
writer.removeNamespace(namespaceName);
} else if (!writer.hasNamespace(namespaceName)) {
// No data in fts db for namespace.
// The namespace could have been removed from fts db
// or the whole fts db have been removed
// without removing it from indexMap.
indexMap.erase(it);
}
} else {
// Needed in case namespaceName was removed from indexMap
// without removing it from fts db.
// May happen when e.g. qch file was removed manually
// without removing fts db.
writer.removeNamespace(namespaceName);
}
// TODO: we may also detect if there are any other data
// and remove it
}
} else {
indexMap.clear();
}
auto it = indexMap.begin();
while (it != indexMap.end()) {
if (!registeredDocs.contains(it.key())) {
writer.removeNamespace(it.key());
it = indexMap.erase(it);
} else {
++it;
}
}
for (const QString &namespaceName : registeredDocs) {
lock.relock();
if (m_cancel) {
// store what we have done so far
writeIndexMap(&engine, indexMap);
writer.endTransaction();
emit indexingFinished();
return;
}
lock.unlock();
// if indexed, continue
if (indexMap.contains(namespaceName))
continue;
const QString fileName = engine.documentationFileName(namespaceName);
QHelpDBReader reader(fileName, QHelpGlobal::uniquifyConnectionName(
fileName, this), nullptr);
if (!reader.init())
continue;
const QString virtualFolder = reader.virtualFolder();
const QList<QStringList> &attributeSets =
engine.filterAttributeSets(namespaceName);
for (const QStringList &attributes : attributeSets) {
const QString &attributesString = attributes.join(u'|');
bool canceled = false;
reader.filesData(attributes, {"html"_L1, "htm"_L1, "txt"_L1},
[&](const QString &file, const QByteArray &data) {
lock.relock();
if (m_cancel) {
canceled = true;
return false;
}
lock.unlock();
if (data.isEmpty())
return true;
QUrl url;
url.setScheme("qthelp"_L1);
url.setAuthority(namespaceName);
url.setPath(u'/' + virtualFolder + u'/' + file);
if (url.hasFragment())
url.setFragment({});
const QString &fullFileName = url.toString();
if (!fullFileName.endsWith(".html"_L1) && !fullFileName.endsWith(".htm"_L1)
&& !fullFileName.endsWith(".txt"_L1)) {
return true;
}
QTextStream s(data);
auto encoding = QStringDecoder::encodingForHtml(data);
if (encoding)
s.setEncoding(*encoding);
const QString &text = s.readAll();
if (text.isEmpty())
return true;
QString title;
QString contents;
if (fullFileName.endsWith(".txt"_L1)) {
title = fullFileName.mid(fullFileName.lastIndexOf(u'/') + 1);
contents = text.toHtmlEscaped();
} else {
#if QT_CONFIG(fullqthelp) && QT_CONFIG(texthtmlparser)
QTextDocument doc;
doc.setHtml(text);
title = doc.metaInformation(QTextDocument::DocumentTitle).toHtmlEscaped();
contents = doc.toPlainText().toHtmlEscaped();
#else
return;
#endif
}
writer.insertDoc(namespaceName, attributesString, fullFileName, title, contents);
return true;
});
if (canceled) {
// store what we have done so far
writeIndexMap(&engine, indexMap);
writer.endTransaction();
emit indexingFinished();
return;
}
}
writer.flush();
const QString &path = engine.documentationFileName(namespaceName);
indexMap.insert(namespaceName, QFileInfo(path).lastModified());
}
writeIndexMap(&engine, indexMap);
writer.endTransaction();
emit indexingFinished();
}
} // namespace fulltextsearch
QT_END_NAMESPACE