Revision control
Copy as Markdown
// Copyright (C) 2020 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
// Qt-Security score:significant reason:default
#include "qqsbcollection_p.h"
#include <QtCore/QLockFile>
#include <QtCore/QSaveFile>
#include <QtCore/QCryptographicHash>
#include <QtCore/QLoggingCategory>
#include <rhi/qrhi.h>
QT_BEGIN_NAMESPACE
Q_STATIC_LOGGING_CATEGORY(lcQuick3DCache, "qt.quick3d.cache")
QQsbCollection::~QQsbCollection()
{
}
QDataStream &operator<<(QDataStream &stream, const QQsbCollection::Entry &entry)
{
return (stream << entry.key << entry.value);
}
QDataStream &operator>>(QDataStream &stream, QQsbCollection::Entry &entry)
{
QByteArray key;
qint64 value;
stream >> key >> value;
entry = QQsbCollection::Entry(key, value);
return stream;
}
size_t qHash(const QQsbCollection::Entry &entry, size_t)
{
return entry.hashKey;
}
bool operator==(const QQsbCollection::Entry &l, const QQsbCollection::Entry &r)
{
return (l.key == r.key);
}
QDataStream &operator<<(QDataStream &stream, const QQsbCollection::EntryDesc &entryDesc)
{
return (stream << entryDesc.materialKey
<< entryDesc.featureSet
<< entryDesc.vertShader.serialized()
<< entryDesc.fragShader.serialized());
}
QDataStream &operator>>(QDataStream &stream, QQsbCollection::EntryDesc &entryDesc)
{
QByteArray desc;
QQsbCollection::FeatureSet fs;
QByteArray vertData;
QByteArray fragData;
stream >> desc >> fs >> vertData >> fragData;
entryDesc.materialKey = desc;
entryDesc.featureSet = fs;
entryDesc.vertShader = QShader::fromSerialized(vertData);
entryDesc.fragShader = QShader::fromSerialized(fragData);
return stream;
}
static constexpr quint64 MagicaDS = 0x3933333335346337;
static constexpr qint64 HeaderSize = sizeof(qint64 /*startOffs*/) + sizeof(quint8 /*version*/) + sizeof(quint32 /*qtVersion*/) + sizeof(MagicaDS);
static constexpr quint32 QtVersion = (QT_VERSION_MAJOR << 16) | (QT_VERSION_MINOR << 8) | (QT_VERSION_PATCH);
bool QQsbCollection::readEndHeader(QDataStream &ds, qint64 *startPos, quint8 *version)
{
quint64 fileId = 0;
quint32 qtver = 0;
ds >> *startPos >> *version >> qtver >> fileId;
if (fileId != MagicaDS) {
qCDebug(lcQuick3DCache, "Corrupt qsbc file");
return false;
}
if (*version != Version::Two) {
qCDebug(lcQuick3DCache, "qsbc file has an unsupported version");
return false;
}
if (qtver != QtVersion) {
qCDebug(lcQuick3DCache, "qsbc file is for a different Qt version");
return false;
}
return true;
}
bool QQsbCollection::readEndHeader(QIODevice *device, EntryMap *entries, quint8 *version)
{
bool result = false;
const qint64 size = device->size();
if (device->seek(size - HeaderSize)) {
QDataStream ds(device);
ds.setVersion(QDataStream::Qt_6_0);
qint64 startPos = 0;
if (readEndHeader(ds, &startPos, version)) {
if (startPos >= 0 && startPos < size && device->seek(startPos)) {
ds >> *entries;
result = true;
}
}
}
return result;
}
void QQsbCollection::writeEndHeader(QDataStream &ds, qint64 startPos, quint8 version, quint64 magic)
{
ds << startPos << version << QtVersion << magic;
}
void QQsbCollection::writeEndHeader(QIODevice *device, const EntryMap &entries)
{
if (!device->atEnd()) {
device->seek(device->size() - 1);
Q_ASSERT(device->atEnd());
}
QDataStream ds(device);
ds.setVersion(QDataStream::Qt_6_0);
const qint64 startPos = device->pos();
ds << entries;
writeEndHeader(ds, startPos, quint8(Version::Two), MagicaDS);
}
QByteArray QQsbCollection::EntryDesc::generateSha(const QByteArray &materialKey, const FeatureSet &featureSet)
{
QCryptographicHash h(QCryptographicHash::Algorithm::Sha1);
h.addData(materialKey);
for (auto it = featureSet.cbegin(), end = featureSet.cend(); it != end; ++it) {
if (it.value())
h.addData(it.key());
}
return h.result().toHex();
}
QByteArray QQsbCollection::EntryDesc::generateSha() const
{
return generateSha(materialKey, featureSet);
}
QQsbCollection::EntryMap QQsbInMemoryCollection::availableEntries() const
{
return EntryMap(entries.keyBegin(), entries.keyEnd());
}
QQsbCollection::Entry QQsbInMemoryCollection::addEntry(const QByteArray &key, const EntryDesc &entryDesc)
{
Entry e(key);
if (!entries.contains(e)) {
entries.insert(e, entryDesc);
return e;
}
return {}; // can only add with a given key once
}
bool QQsbInMemoryCollection::extractEntry(Entry entry, EntryDesc &entryDesc)
{
auto it = entries.constFind(entry);
if (it != entries.constEnd()) {
entryDesc = *it;
return true;
}
return false;
}
void QQsbInMemoryCollection::clear()
{
entries.clear();
}
static inline QString lockFileName(const QString &name)
{
return name + QLatin1String(".lck");
}
bool QQsbInMemoryCollection::load(const QString &filename)
{
QLockFile lock(lockFileName(filename));
if (!lock.lock()) {
qCDebug(lcQuick3DCache, "Could not create shader cache lock file '%s'",
qPrintable(lock.fileName()));
return false;
}
QFile f(filename);
if (!f.open(QIODevice::ReadOnly)) {
qCDebug(lcQuick3DCache, "Failed to open qsbc file %s", qPrintable(filename));
return false;
}
EntryMap entryMap;
quint8 version = 0;
if (!readEndHeader(&f, &entryMap, &version)) {
qCDebug(lcQuick3DCache, "Ignoring qsbc file %s", qPrintable(filename));
return false;
}
f.seek(0);
const qint64 size = f.size();
clear();
const qsizetype entryCountBefore = entries.size();
for (const Entry &e : std::as_const(entryMap)) {
const qint64 offset = e.value;
if (e.isValid() && offset >= 0 && size > offset && f.seek(offset)) {
QDataStream ds(&f);
ds.setVersion(QDataStream::Qt_6_0);
EntryDesc entryDesc;
ds >> entryDesc;
entries.insert(Entry(e.key), entryDesc);
}
}
qCDebug(lcQuick3DCache, "Loaded %d material cache entries from %s",
int(entries.size() - entryCountBefore),
qPrintable(filename));
return true;
}
bool QQsbInMemoryCollection::save(const QString &filename)
{
QLockFile lock(lockFileName(filename));
if (!lock.lock()) {
qCDebug(lcQuick3DCache, "Could not create shader cache lock file '%s'",
qPrintable(lock.fileName()));
return false;
}
#if QT_CONFIG(temporaryfile)
QSaveFile f(filename);
#else
QFile f(filename);
#endif
if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
qCDebug(lcQuick3DCache, "Failed to write qsbc file %s", qPrintable(filename));
return false;
}
QDataStream ds(&f);
ds.setVersion(QDataStream::Qt_6_0);
EntryMap entryMap;
for (auto it = entries.cbegin(), end = entries.cend(); it != end; ++it) {
const qint64 offset = f.pos();
ds << it.value();
entryMap.insert(Entry(it.key().key, offset));
}
writeEndHeader(&f, entryMap);
qCDebug(lcQuick3DCache, "Writing %d material cache entries to %s",
int(entryMap.size()),
qPrintable(filename));
#if QT_CONFIG(temporaryfile)
return f.commit();
#else
return true;
#endif
}
QQsbIODeviceCollection::QQsbIODeviceCollection(const QString &filePath)
: file(filePath)
, device(file)
{
}
QQsbIODeviceCollection::QQsbIODeviceCollection(QIODevice &dev)
: device(dev)
, devOwner(DeviceOwner::Extern)
{
}
QQsbIODeviceCollection::~QQsbIODeviceCollection()
{
if (!entries.isEmpty() || device.isOpen())
unmap();
}
bool QQsbIODeviceCollection::map(MapMode mode)
{
if (device.isOpen()) {
// Make sure Truncate is set if we're writing.
if ((device.openMode() & QIODevice::WriteOnly) != 0) {
if ((device.openMode() & QIODevice::Truncate) == 0) {
qWarning("Open mode needs to have Truncate set for writing!");
return false;
}
if ((device.openMode() & QIODevice::Text) != 0) {
qWarning("Open mode can't have Text mode set!");
return false;
}
}
} else if (!device.open(QIODevice::OpenMode(mode))) {
qWarning("Unable to open device!");
return false;
}
if (mode == Write)
return true;
Q_ASSERT(mode == Read);
const bool ret = readEndHeader(&device, &entries, &version);
if (!ret)
unmap();
return ret;
}
void QQsbIODeviceCollection::unmap()
{
if (device.isOpen() && ((device.openMode() & Write) == Write)) {
if (!entries.isEmpty()) {
writeEndHeader(&device, entries);
} else {
if (devOwner == DeviceOwner::Self)
file.remove();
}
}
device.close();
entries.clear();
}
QQsbCollection::EntryMap QQsbIODeviceCollection::availableEntries() const
{
return entries;
}
QQsbCollection::Entry QQsbIODeviceCollection::addEntry(const QByteArray &key, const EntryDesc &entryDesc)
{
if (entries.contains(Entry(key)) || !map(MapMode::Write))
return {};
QDataStream ds(&device);
ds.setVersion(QDataStream::Qt_6_0);
const auto offset = device.pos();
ds << entryDesc;
Entry e(key, offset);
entries.insert(e);
return e;
}
bool QQsbIODeviceCollection::extractEntry(Entry entry, EntryDesc &entryDesc)
{
if (device.isOpen() && device.isReadable()) {
const qint64 offset = entry.value;
if (entry.isValid() && offset >= 0) {
const qint64 size = device.size();
if (size > offset && device.seek(offset)) {
QDataStream ds(&device);
ds.setVersion(QDataStream::Qt_6_0);
ds >> entryDesc;
return true;
}
} else {
qWarning("Entry not found id(%s), offset(%lld)", entry.key.constData(), entry.value);
}
} else {
qWarning("Unable to open file for reading");
}
return false;
}
static const char *borderText() { return "--------------------------------------------------------------------------------"; }
void QQsbIODeviceCollection::dumpInfo()
{
if (map(QQsbIODeviceCollection::Read)) {
qDebug("Number of entries in collection: %zu\n", size_t(entries.size()));
int i = 0;
qDebug("Qsbc version: %u", version);
for (const auto &e : std::as_const(entries)) {
qDebug("%s\n"
"Entry %d\n%s\n"
"Key: %s\n"
"Offset: %llu", borderText(), i++, borderText(), e.key.constData(), e.value);
QQsbCollection::EntryDesc ed;
if (extractEntry(e, ed)) {
qDebug() << ed.materialKey << Qt::endl
<< ed.featureSet << Qt::endl
<< ed.vertShader << Qt::endl
<< ed.fragShader;
} else {
qWarning("Extracting Qsb entry failed!");
}
}
}
unmap();
}
void QQsbIODeviceCollection::dumpInfo(const QString &file)
{
QQsbIODeviceCollection qsbc(file);
qsbc.dumpInfo();
}
void QQsbIODeviceCollection::dumpInfo(QIODevice &device)
{
QQsbIODeviceCollection qsbc(device);
qsbc.dumpInfo();
}
QT_END_NAMESPACE