Revision control
Copy as Markdown
// Copyright (C) 2025 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
// Qt-Security score:critical reason:data-parser
#include "qssglightmapio_p.h"
#include <private/qssgrenderloadedtexture_p.h>
#include <private/qssgassert_p.h>
#include <QDataStream>
#include <QDebug>
#include <QtEndian>
#include <QFile>
#include <algorithm>
#include <cstring>
QT_BEGIN_NAMESPACE
using IndexKey = std::tuple<QSSGLightmapIODataTag /* dataTag */, qint32 /* keySize */, QByteArray /* key */>;
struct IndexEntry
{
qint64 keyOffset; // 8 bytes
qint64 keySize; // 8 bytes
qint64 dataOffset; // 8 bytes
qint64 dataSize; // 8 bytes
QSSGLightmapIODataTag dataTag; // 4 bytes
quint32 padding;
};
struct QSSGLightmapIOPrivate
{
QByteArray readKey(const IndexKey &indexKey) const;
bool writeHeader() const;
bool writeData(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &data);
bool writeFooter();
bool decodeHeaders();
QList<std::pair<QString, QSSGLightmapIODataTag>> getKeys() const;
QSharedPointer<QIODevice> stream;
QMap<IndexKey, IndexEntry> entries; // Maps from name -> entry
qint64 entryCount = -1;
qint64 indexOffset = -1;
qint64 fileVersion = -1;
qint64 fileSize = -1;
};
static_assert(offsetof(IndexEntry, keyOffset) == 0, "Unexpected alignment");
static_assert(offsetof(IndexEntry, keySize) == 8, "Unexpected alignment");
static_assert(offsetof(IndexEntry, dataOffset) == 16, "Unexpected alignment");
static_assert(offsetof(IndexEntry, dataSize) == 24, "Unexpected alignment");
static_assert(offsetof(IndexEntry, dataTag) == 32, "Unexpected alignment");
static_assert(offsetof(IndexEntry, padding) == 36, "Unexpected alignment");
static_assert(sizeof(IndexEntry) == 40, "Unexpected size");
constexpr char fileSignature[] = "QTLTMP";
static IndexKey keyToIndexKey(const QString &key, QSSGLightmapIODataTag tag)
{
QByteArray keyBuffer = key.toUtf8();
return std::make_tuple(tag, keyBuffer.size(), keyBuffer);
}
static QByteArray mapToByteArray(const QVariantMap &map)
{
QByteArray byteArray;
QDataStream out(&byteArray, QIODevice::WriteOnly);
out.setVersion(QDataStream::Qt_6_9);
out.setByteOrder(QDataStream::LittleEndian);
out << QVariant(map);
return byteArray;
}
static void convertEndian(QByteArray &buffer, int sizeOfDataType)
{
if (QSysInfo::ByteOrder == QSysInfo::LittleEndian) {
return; // No need to flip
} else if (QSysInfo::ByteOrder == QSysInfo::BigEndian) {
// Need to flip
} else {
qWarning() << "QSSGLightmapLoader::convertEndian: Unknown endianness";
return;
}
Q_ASSERT(buffer.size() % sizeOfDataType == 0);
if (buffer.size() % sizeOfDataType != 0) {
qWarning() << "QSSGLightmapLoader::convertEndian: Unexpected buffer size";
return;
}
if (sizeOfDataType == 1)
return;
const int buffSize = buffer.size();
for (int offset = 0; offset < buffSize; offset += sizeOfDataType) {
for (int i = 0; i < sizeOfDataType / 2; ++i) {
std::swap(buffer[offset + i], buffer[offset + sizeOfDataType - 1 - i]);
}
}
}
static QVariantMap byteArrayToMap(QByteArray input)
{
QVariant variant;
QDataStream in(input);
in.setVersion(QDataStream::Qt_6_9);
in.setByteOrder(QDataStream::LittleEndian);
in >> variant;
return variant.toMap();
}
bool QSSGLightmapIOPrivate::decodeHeaders()
{
Q_ASSERT(stream);
if (!stream->isOpen())
stream->open(QIODeviceBase::OpenModeFlag::ReadOnly);
if (!stream->isOpen()) {
qWarning() << "QSSGLightmapIO: Stream is not openable";
return false;
}
if (!stream->isReadable()) {
qWarning() << "QSSGLightmapIO: Stream is not readable";
return false;
}
// [Magic Bytes][Version]
constexpr int headerSize = 6 * sizeof(char) + sizeof(qint32);
// [Entry Count][Offset Index]
constexpr int footerSize = 2 * sizeof(qint64);
const qint64 fileSize = stream->size();
if (fileSize < headerSize + footerSize) {
qWarning() << "QSSGLightmapIO: File too small to contain header and footer";
return false;
}
stream->seek(0);
QByteArray headerData = stream->read(headerSize);
if (headerData.size() != qsizetype(headerSize)) {
qWarning() << "Failed to read header";
return false;
}
// Verify file signature
if (QByteArrayView(headerData.constData(), 6) != QByteArray::fromRawData(fileSignature, 6)) {
qWarning() << "QSSGLightmapIO: Invalid file signature";
return false;
}
qint32 fileVersion = -1;
// Read file version (4 bytes after 6 magic bytes)
const char *versionPtr = headerData.constData() + 6;
fileVersion = qFromLittleEndian<qint32>(versionPtr);
if (fileVersion != 1) {
qWarning() << "QSSGLightmapIO: Invalid file version";
return false;
}
// Seek to the last bytes (footer)
if (!stream->seek(fileSize - footerSize)) {
qWarning() << "Failed to seek to footer";
return false;
}
QByteArray footerData = stream->read(footerSize);
if (footerData.size() != qsizetype(footerSize)) {
qWarning() << "Failed to read footer";
return false;
}
const char *footerPtr = footerData.constData();
const qint64 indexOffset = qFromLittleEndian<qint64>(footerPtr);
const qint64 entryCount = qFromLittleEndian<qint64>(footerPtr + sizeof(qint64));
this->entryCount = entryCount;
this->indexOffset = indexOffset;
this->fileVersion = fileVersion;
this->fileSize = fileSize;
return true;
}
bool QSSGLightmapIOPrivate::writeHeader() const
{
Q_ASSERT(stream);
if (!stream->isOpen())
stream->open(QIODeviceBase::OpenModeFlag::WriteOnly);
if (!stream->isOpen()) {
qWarning() << "QSSGLightmapIO: File is not openable";
return false;
}
if (!stream->isWritable()) {
qWarning() << "QSSGLightmapIO: File is not writable";
return false;
}
stream->seek(0);
// Write file signature
if (stream->write(fileSignature, sizeof(fileSignature) - 1) != sizeof(fileSignature) - 1) {
qWarning() << "QSSGLightmapIO: Failed to write file signature";
stream->close();
return false;
}
// Write file version
const qint32 version = qToLittleEndian<qint32>(1);
if (stream->write(reinterpret_cast<const char *>(&version), sizeof(version)) != sizeof(version)) {
qWarning() << "QSSGLightmapIO: Failed to write file version";
stream->close();
return false;
}
return true;
}
bool QSSGLightmapIOPrivate::writeData(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &data)
{
Q_ASSERT(stream->isOpen() && stream->isWritable());
IndexKey keyBytes = keyToIndexKey(key, tag);
Q_ASSERT(!entries.contains(keyBytes));
IndexEntry &entry = entries[keyBytes];
entry.dataOffset = stream->pos();
entry.dataSize = data.size();
entry.dataTag = tag;
if (stream->write(data) != data.size()) {
qWarning() << "QSSGLightmapIO: Failed to write entry data";
return false;
}
return true;
}
template<typename T>
bool writeType(const QSharedPointer<QIODevice> &stream, T value)
{
return stream->write(reinterpret_cast<const char *>(&value), sizeof(value)) == sizeof(value);
}
bool QSSGLightmapIOPrivate::writeFooter()
{
Q_ASSERT(stream);
Q_ASSERT(stream->isOpen());
Q_ASSERT(stream->isWritable());
// Store the key strings
for (auto it = entries.begin(); it != entries.end(); ++it) {
auto [dataTag, keySize, key] = it.key();
IndexEntry &entry = it.value();
entry.keyOffset = stream->pos();
entry.keySize = keySize;
if (stream->write(key) != key.size()) {
qWarning() << "QSSGLightmapIO: Failed to write key";
return false;
}
}
// The file should be seeked to the end of the data segment so we can just
// add indices and footer now
const qint64 indexOffset = qToLittleEndian<qint64>(stream->pos());
const qint64 indexCount = qToLittleEndian<qint64>(entries.size());
for (const IndexEntry &entry : std::as_const(entries)) {
const qint64 keyOffset = qToLittleEndian<qint64>(entry.keyOffset);
const qint64 keySize = qToLittleEndian<qint64>(entry.keySize);
const qint64 dataOffset = qToLittleEndian<qint64>(entry.dataOffset);
const qint64 dataSize = qToLittleEndian<qint64>(entry.dataSize);
const quint32 dataTag = qToLittleEndian<quint32>(std::underlying_type_t<QSSGLightmapIODataTag>(entry.dataTag));
const quint32 padding = qToLittleEndian<quint32>(entry.padding);
if (!writeType(stream, keyOffset) || !writeType(stream, keySize) || !writeType(stream, dataOffset)
|| !writeType(stream, dataSize) || !writeType(stream, dataTag) || !writeType(stream, padding)) {
qWarning() << "QSSGLightmapIO: Failed to write entry";
return false;
}
}
// Write footer
if (!writeType(stream, indexOffset) || !writeType(stream, indexCount)) {
qWarning() << "QSSGLightmapIO: Failed to write footer";
return false;
}
stream->close();
return true;
}
QByteArray QSSGLightmapIOPrivate::readKey(const IndexKey &indexKey) const
{
Q_ASSERT(stream);
Q_ASSERT(entryCount >= 0);
Q_ASSERT(indexOffset >= 0);
Q_ASSERT(fileVersion >= 0);
Q_ASSERT(fileSize >= 0);
// Perform binary search by reading one IndexEntry at a time
qint64 low = 0;
qint64 high = entryCount;
IndexEntry entry;
auto [dataTag, keySize, key] = indexKey;
bool found = false;
qint64 matchOffset = 0;
qint64 matchSize = 0;
while (low < high) {
qint64 mid = (low + high) / 2;
qint64 offset = indexOffset + mid * sizeof(IndexEntry);
if (!stream->seek(offset)) {
qWarning() << "Failed to seek to index entry";
return {};
}
if (stream->read(reinterpret_cast<char *>(&entry), sizeof(IndexEntry)) != sizeof(IndexEntry)) {
qWarning() << "Failed to read index entry";
return {};
}
// Sort by dataTag, keySize and name (matching the order of IndexKey)
int cmp = qint64(entry.dataTag) - qint64(dataTag);
if (cmp == 0) {
cmp = entry.keySize - keySize;
}
if (cmp == 0) {
if (!stream->seek(entry.keyOffset)) {
qWarning() << "Failed to seek to key entry";
return {};
}
const QByteArray entryKey = stream->read(entry.keySize);
if (entryKey.size() != entry.keySize) {
qWarning() << "Failed to read to key entry";
return {};
}
for (int i = 0, n = entry.keySize; i < n; ++i) {
cmp = int(entryKey[i]) - int(key[i]);
if (cmp != 0)
break;
}
}
if (cmp < 0) {
low = mid + 1;
} else if (cmp > 0) {
high = mid;
} else {
// Found
found = true;
matchOffset = qFromLittleEndian(entry.dataOffset);
matchSize = qFromLittleEndian(entry.dataSize);
break;
}
}
if (!found) {
qWarning() << "Key not found:" << key;
return {};
}
if (matchOffset + matchSize > fileSize) {
qWarning() << "Asset data out of bounds";
return {};
}
// Seek and read asset
if (!stream->seek(matchOffset)) {
qWarning() << "Failed to seek to asset data";
return {};
}
QByteArray assetData = stream->read(matchSize);
if (assetData.size() != static_cast<int>(matchSize)) {
qWarning() << "Failed to read full asset data";
return {};
}
return assetData;
}
QList<std::pair<QString, QSSGLightmapIODataTag>> QSSGLightmapIOPrivate::getKeys() const
{
Q_ASSERT(stream);
Q_ASSERT(entryCount >= 0);
Q_ASSERT(indexOffset >= 0);
Q_ASSERT(fileVersion >= 0);
Q_ASSERT(fileSize >= 0);
QList<std::pair<QString, QSSGLightmapIODataTag>> keys;
keys.resize(entryCount);
IndexEntry entry;
for (int i = 0; i < entryCount; ++i) {
const qint64 offset = indexOffset + i * sizeof(IndexEntry);
if (!stream->seek(offset)) {
qWarning() << "Failed to seek to index entry";
return {};
}
if (stream->read(reinterpret_cast<char *>(&entry), sizeof(IndexEntry)) != sizeof(IndexEntry)) {
qWarning() << "Failed to read index entry";
return {};
}
if (!stream->seek(entry.keyOffset)) {
qWarning() << "Failed to seek to key entry";
return {};
}
const QByteArray entryKey = stream->read(entry.keySize);
if (entryKey.size() != entry.keySize) {
qWarning() << "Failed to read to key entry";
return {};
}
keys[i] = std::make_pair(QString::fromUtf8(entryKey), static_cast<QSSGLightmapIODataTag>(entry.dataTag));
}
std::sort(keys.begin(), keys.end(), [](const auto &a, const auto &b) { return a.first < b.first; });
return keys;
}
/////////////////////////////// Loader /////////////////////////////////
QSSGLightmapLoader::QSSGLightmapLoader() : d(new QSSGLightmapIOPrivate) { };
QSSGLightmapLoader::~QSSGLightmapLoader()
{
if (d->stream)
d->stream->close();
delete d;
}
QSharedPointer<QSSGLightmapLoader> QSSGLightmapLoader::open(const QSharedPointer<QIODevice> &stream)
{
if (!stream) {
qWarning() << "Failed to open lightmap: invalid stream";
return nullptr;
}
auto loader = QSharedPointer<QSSGLightmapLoader>(new QSSGLightmapLoader);
loader->d->stream = stream;
if (!loader->d->decodeHeaders()) {
if (loader->d->stream->isOpen())
loader->d->stream->close();
return {};
}
return loader;
}
QSharedPointer<QSSGLightmapLoader> QSSGLightmapLoader::open(const QString &path)
{
if (QSharedPointer<QIODevice> source = QSSGInputUtil::getStreamForFile(path))
return open(source);
return nullptr;
}
QByteArray QSSGLightmapLoader::readF32Image(const QString &key, QSSGLightmapIODataTag tag) const
{
QByteArray buffer = d->readKey(keyToIndexKey(key, tag));
convertEndian(buffer, sizeof(float));
return buffer;
}
QByteArray QSSGLightmapLoader::readU32Image(const QString &key, QSSGLightmapIODataTag tag) const
{
QByteArray buffer = d->readKey(keyToIndexKey(key, tag));
convertEndian(buffer, sizeof(quint32));
return buffer;
}
QByteArray QSSGLightmapLoader::readData(const QString &key, QSSGLightmapIODataTag tag) const
{
return d->readKey(keyToIndexKey(key, tag));
}
QVariantMap QSSGLightmapLoader::readMap(const QString &key, QSSGLightmapIODataTag tag) const
{
QByteArray metadataBuffer = d->readKey(keyToIndexKey(key, tag));
QVariantMap metadata = byteArrayToMap(metadataBuffer);
return metadata;
}
QList<std::pair<QString, QSSGLightmapIODataTag>> QSSGLightmapLoader::getKeys() const
{
return d->getKeys();
}
inline int calculateLine(int width, int bitdepth)
{
return ((width * bitdepth) + 7) / 8;
}
inline int calculatePitch(int line)
{
return (line + 3) & ~3;
}
QSSGLoadedTexture *QSSGLightmapLoader::createTexture(QSharedPointer<QIODevice> stream,
const QSSGRenderTextureFormat &format,
const QString &key)
{
QSharedPointer<QSSGLightmapLoader> loader = QSSGLightmapLoader::open(stream);
if (!loader)
return nullptr;
QVariantMap metadata = loader->readMap(key, QSSGLightmapIODataTag::Metadata);
if (metadata.isEmpty())
return nullptr;
bool ok = true;
const int w = metadata[QStringLiteral("width")].toInt(&ok);
Q_ASSERT(ok);
const int h = metadata[QStringLiteral("height")].toInt(&ok);
Q_ASSERT(ok);
QSize pixelSize = QSize(w, h);
QByteArray imageFP32 = loader->readF32Image(key, QSSGLightmapIODataTag::Texture_Final);
if (imageFP32.isEmpty())
return nullptr;
// Setup Output container
const int bytesPerPixel = format.getSizeofFormat();
const int bitCount = bytesPerPixel * 8;
const int pitch = calculatePitch(calculateLine(pixelSize.width(), bitCount));
const size_t dataSize = pixelSize.height() * pitch;
QSSG_CHECK_X(dataSize <= std::numeric_limits<quint32>::max(), "Requested data size exceeds 4GB limit!");
QSSGLoadedTexture *imageData = new QSSGLoadedTexture;
imageData->dataSizeInBytes = quint32(dataSize);
imageData->data = ::malloc(imageData->dataSizeInBytes);
imageData->width = pixelSize.width();
imageData->height = pixelSize.height();
imageData->format = format;
imageData->components = format.getNumberOfComponent();
imageData->isSRGB = false;
std::array<float, 4> *source = reinterpret_cast<std::array<float, 4> *>(imageFP32.data());
quint8 *target = reinterpret_cast<quint8 *>(imageData->data);
int idx = 0;
float rgbaF32[4];
for (int y = imageData->height - 1; y >= 0; --y) {
for (int x = 0; x < imageData->width; x++) {
const int i = y * imageData->width + x;
const int lh = i * 4 * sizeof(float);
const int rh = imageFP32.size();
Q_ASSERT(lh < rh);
std::array<float, 4> v = source[i];
rgbaF32[0] = v[0];
rgbaF32[1] = v[1];
rgbaF32[2] = v[2];
rgbaF32[3] = v[3];
format.encodeToPixel(rgbaF32, target, idx * bytesPerPixel);
++idx;
}
}
return imageData;
}
///////////////////////////// Writer ///////////////////////////////////
QSSGLightmapWriter::QSSGLightmapWriter() : d(new QSSGLightmapIOPrivate) { }
QSSGLightmapWriter::~QSSGLightmapWriter()
{
if (d->stream->isOpen()) {
qWarning() << "Lightmap file open on destruction, closing";
close();
}
delete d;
}
QSharedPointer<QSSGLightmapWriter> QSSGLightmapWriter::open(const QString &path)
{
return open(QSharedPointer<QIODevice>(new QFile(path)));
}
QSharedPointer<QSSGLightmapWriter> QSSGLightmapWriter::open(const QSharedPointer<QIODevice> &stream)
{
if (!stream) {
qWarning() << "Failed to open lightmap file";
return nullptr;
}
QSharedPointer<QSSGLightmapWriter> writer = QSharedPointer<QSSGLightmapWriter>(new QSSGLightmapWriter);
if (!stream->isOpen() && !stream->open(QIODeviceBase::WriteOnly)) {
qWarning() << "Failed to open lightmap file";
return nullptr;
}
writer->d->stream = stream;
if (!writer->d->writeHeader())
return nullptr;
return writer;
}
bool QSSGLightmapWriter::writeF32Image(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &imageFP32)
{
QByteArray buffer = QByteArray(imageFP32.constData(), imageFP32.size());
convertEndian(buffer, sizeof(float));
return d->writeData(key, tag, buffer);
}
bool QSSGLightmapWriter::writeU32Image(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &imageU32)
{
QByteArray buffer = QByteArray(imageU32.constData(), imageU32.size());
convertEndian(buffer, sizeof(quint32));
return d->writeData(key, tag, buffer);
}
bool QSSGLightmapWriter::writeData(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &buffer)
{
return d->writeData(key, tag, buffer);
}
bool QSSGLightmapWriter::writeMap(const QString &key, QSSGLightmapIODataTag tag, const QVariantMap &data)
{
return d->writeData(key, tag, mapToByteArray(data));
}
bool QSSGLightmapWriter::close() const
{
return d->writeFooter();
}
QT_END_NAMESPACE