Revision control
++ // Copyright (C) 2020 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++ // Qt-Security score:critical reason:data-parser
++
++ #include "qbinaryjson_p.h"
++
++ #include <QtCore/qjsonobject.h>
++ #include <QtCore/qjsonarray.h>
++ #include <QtCore/qscopeguard.h>
++ #include <QtCore/qvarlengtharray.h>
++
++ #include <private/qbinaryjsonarray_p.h>
++ #include <private/qbinaryjsonobject_p.h>
++
++ QT_BEGIN_NAMESPACE
++
++ /*!
++ \namespace QBinaryJson
++ \inmodule QtCore5Compat
++ \brief Contains functions for converting QJsonDocument to and from JSON binary format.
++
++ This namespace provides utility functions to keep compatibility with
++ older code, which uses the JSON binary format for serializing JSON. Qt JSON
++ types can be converted to Qt CBOR types, which can in turn be serialized
++ into the CBOR binary format and vice versa.
++ */
++
++ /*! \enum QBinaryJson::DataValidation
++
++ This enum is used to tell QJsonDocument whether to validate the binary data
++ when converting to a QJsonDocument using fromBinaryData() or fromRawData().
++
++ \value Validate Validate the data before using it. This is the default.
++ \value BypassValidation Bypasses data validation. Only use if you received the
++ data from a trusted place and know it's valid, as using of invalid data can crash
++ the application.
++ */
++
++ namespace QBinaryJson {
++
++ /*!
++ Creates a QJsonDocument that uses the first \a size bytes from
++ \a data. It assumes \a data contains a binary encoded JSON document.
++ The created document does not take ownership of \a data. The data is
++ copied into a different data structure, and the original data can be
++ deleted or modified afterwards.
++
++ \a data has to be aligned to a 4 byte boundary.
++
++ \a validation decides whether the data is checked for validity before being used.
++ By default the data is validated. If the \a data is not valid, the method returns
++ a null document.
++
++ Returns a QJsonDocument representing the data.
++
++ \note The binary JSON encoding is only retained for backwards
++ compatibility. It is undocumented and restrictive in the maximum size of JSON
++ documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
++ which can in turn be serialized into the CBOR binary format and vice versa. The
++ CBOR format is a well-defined and less restrictive binary representation for a
++ superset of JSON.
++
++ \note Before Qt 5.15, the caller had to guarantee that \a data would not be
++ deleted or modified as long as any QJsonDocument, QJsonObject or QJsonArray
++ still referenced the data. From Qt 5.15 on, this is not necessary anymore.
++
++ \sa toRawData(), fromBinaryData(), DataValidation, QCborValue
++ */
++ QJsonDocument fromRawData(const char *data, int size, DataValidation validation)
++ {
++ if (quintptr(data) & 3) {
++ qWarning("QJsonDocument::fromRawData: data has to have 4 byte alignment");
++ return QJsonDocument();
++ }
++
++ if (size < 0 || uint(size) < sizeof(QBinaryJsonPrivate::Header) + sizeof(QBinaryJsonPrivate::Base))
++ return QJsonDocument();
++
++ std::unique_ptr<QBinaryJsonPrivate::ConstData> binaryData
++ = std::make_unique<QBinaryJsonPrivate::ConstData>(data, size);
++
++ return (validation == BypassValidation || binaryData->isValid())
++ ? binaryData->toJsonDocument()
++ : QJsonDocument();
++ }
++
++ static char *toRawDataHelper(const QJsonDocument &document, int *size)
++ {
++ if (document.isNull()) {
++ *size = 0;
++ return nullptr;
++ }
++
++ char *rawData = nullptr;
++ uint rawDataSize = 0;
++ if (document.isObject()) {
++ QBinaryJsonObject o = QBinaryJsonObject::fromJsonObject(document.object());
++ rawData = o.takeRawData(&rawDataSize);
++ } else {
++ QBinaryJsonArray a = QBinaryJsonArray::fromJsonArray(document.array());
++ rawData = a.takeRawData(&rawDataSize);
++ }
++
++ // It would be quite miraculous if not, as we should have hit the 128MB limit then.
++ Q_ASSERT(rawDataSize <= uint(std::numeric_limits<int>::max()));
++
++ *size = static_cast<int>(rawDataSize);
++ return rawData;
++ }
++
++ #if QT_DEPRECATED_SINCE(6, 12)
++ /*!
++ \deprecated [6.12] Use toBinaryData() instead.
++
++ Returns the raw binary representation of \a document.
++ \a size will contain the size of the returned data.
++
++ This method is useful to e.g. stream the JSON document
++ in its binary form to a file.
++
++ \warning In Qt 6, unlike Qt 5, the caller takes ownership of the returned
++ data and must release it with \c{free()} once it is not needed anymore.
++ Use toBinaryData() to get the binary representation in a QByteArray, which
++ manages the memory automatically.
++
++ \note The binary JSON encoding is only retained for backwards
++ compatibility. It is undocumented and restrictive in the maximum size of JSON
++ documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
++ which can in turn be serialized into the CBOR binary format and vice versa. The
++ CBOR format is a well-defined and less restrictive binary representation for a
++ superset of JSON.
++
++ \sa fromRawData(), fromBinaryData(), toBinaryData(), QCborValue
++ */
++ const char *toRawData(const QJsonDocument &document, int *size)
++ {
++ qWarning("QBinaryJson: In Qt 6, unlike Qt 5, toRawData() transfers ownership "
++ "of the pointer to the caller. Prefer toBinaryData() instead.");
++ return toRawDataHelper(document, size);
++ }
++ #endif // QT_DEPRECATED_SINCE(6, 12)
++
++ /*!
++ Creates a QJsonDocument from \a data.
++
++ \a validation decides whether the data is checked for validity before being used.
++ By default the data is validated. If the \a data is not valid, the method returns
++ a null document.
++
++ \note The binary JSON encoding is only retained for backwards
++ compatibility. It is undocumented and restrictive in the maximum size of JSON
++ documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
++ which can in turn be serialized into the CBOR binary format and vice versa. The
++ CBOR format is a well-defined and less restrictive binary representation for a
++ superset of JSON.
++
++ \sa toBinaryData(), fromRawData(), DataValidation, QCborValue
++ */
++ QJsonDocument fromBinaryData(const QByteArray &data, DataValidation validation)
++ {
++ if (size_t(data.size()) < sizeof(QBinaryJsonPrivate::Header) + sizeof(QBinaryJsonPrivate::Base))
++ return QJsonDocument();
++
++ QBinaryJsonPrivate::Header h;
++ memcpy(&h, data.constData(), sizeof(QBinaryJsonPrivate::Header));
++ QBinaryJsonPrivate::Base root;
++ memcpy(&root, data.constData() + sizeof(QBinaryJsonPrivate::Header),
++ sizeof(QBinaryJsonPrivate::Base));
++
++ uint size = 0;
++ if (qAddOverflow(uint{sizeof(QBinaryJsonPrivate::Header)}, uint{root.size}, &size))
++ return QJsonDocument(); // not representable in Private::ConstData
++
++ if (h.tag != QJsonDocument::BinaryFormatTag || h.version != 1U || size > size_t(data.size()))
++ return QJsonDocument();
++
++ std::unique_ptr<QBinaryJsonPrivate::ConstData> d
++ = std::make_unique<QBinaryJsonPrivate::ConstData>(data.constData(), size);
++
++ return (validation == BypassValidation || d->isValid())
++ ? d->toJsonDocument()
++ : QJsonDocument();
++ }
++
++ /*!
++ Returns a binary representation of \a document.
++
++ The binary representation is also the native format used internally in Qt,
++ and is very efficient and fast to convert to and from.
++
++ The binary format can be stored on disk and interchanged with other applications
++ or computers. fromBinaryData() can be used to convert it back into a
++ JSON document.
++
++ \note The binary JSON encoding is only retained for backwards
++ compatibility. It is undocumented and restrictive in the maximum size of JSON
++ documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
++ which can in turn be serialized into the CBOR binary format and vice versa. The
++ CBOR format is a well-defined and less restrictive binary representation for a
++ superset of JSON.
++
++ \sa fromBinaryData(), QCborValue
++ */
++ QByteArray toBinaryData(const QJsonDocument &document)
++ {
++ int size = 0;
++ char *raw = toRawDataHelper(document, &size);
++ // We have to cleanup the raw data, because QBA makes a copy of it.
++ const auto releaseRawData = qScopeGuard([raw]() {
++ free(raw);
++ });
++ return QByteArray(raw, size);
++ }
++
++ } // namespace QBinaryJson
++
++ namespace QBinaryJsonPrivate {
++
++ static Q_CONSTEXPR Base emptyArray = {
++ { qle_uint(sizeof(Base)) },
++ { 0 },
++ { qle_uint(0) }
++ };
++
++ static Q_CONSTEXPR Base emptyObject = {
++ { qle_uint(sizeof(Base)) },
++ { qToLittleEndian(1U) },
++ { qle_uint(0) }
++ };
++
++ void MutableData::compact()
++ {
++ static_assert(sizeof(Value) == sizeof(offset));
++
++ Base *base = header->root();
++ int reserve = 0;
++ if (base->isObject()) {
++ auto *o = static_cast<Object *>(base);
++ for (uint i = 0; i < o->length(); ++i)
++ reserve += o->entryAt(i)->usedStorage(o);
++ } else {
++ auto *a = static_cast<Array *>(base);
++ for (uint i = 0; i < a->length(); ++i)
++ reserve += a->at(i)->usedStorage(a);
++ }
++
++ uint size = sizeof(Base) + reserve + base->length() * sizeof(offset);
++ uint alloc = sizeof(Header) + size;
++ auto *h = reinterpret_cast<Header *>(malloc(alloc));
++ Q_CHECK_PTR(h);
++ h->tag = QJsonDocument::BinaryFormatTag;
++ h->version = 1;
++ Base *b = new (h->root()) Base{};
++ b->size = size;
++ if (header->root()->isObject())
++ b->setIsObject();
++ else
++ b->setIsArray();
++ b->setLength(base->length());
++ b->tableOffset = reserve + sizeof(Array);
++
++ uint offset = sizeof(Base);
++ if (b->isObject()) {
++ const auto *o = static_cast<const Object *>(base);
++ auto *no = static_cast<Object *>(b);
++
++ for (uint i = 0; i < o->length(); ++i) {
++ no->table()[i] = offset;
++
++ const Entry *e = o->entryAt(i);
++ Entry *ne = no->entryAt(i);
++ uint s = e->size();
++ memcpy(ne, e, s);
++ offset += s;
++ uint dataSize = e->value.usedStorage(o);
++ if (dataSize) {
++ memcpy(reinterpret_cast<char *>(no) + offset, e->value.data(o), dataSize);
++ ne->value.setValue(offset);
++ offset += dataSize;
++ }
++ }
++ } else {
++ const auto *a = static_cast<const Array *>(base);
++ auto *na = static_cast<Array *>(b);
++
++ for (uint i = 0; i < a->length(); ++i) {
++ const Value *v = a->at(i);
++ Value *nv = na->at(i);
++ *nv = *v;
++ uint dataSize = v->usedStorage(a);
++ if (dataSize) {
++ memcpy(reinterpret_cast<char *>(na) + offset, v->data(a), dataSize);
++ nv->setValue(offset);
++ offset += dataSize;
++ }
++ }
++ }
++ Q_ASSERT(offset == uint(b->tableOffset));
++
++ free(header);
++ header = h;
++ this->alloc = alloc;
++ compactionCounter = 0;
++ }
++
++ bool ConstData::isValid() const
++ {
++ if (header->tag != QJsonDocument::BinaryFormatTag || header->version != 1U)
++ return false;
++
++ const Base *root = header->root();
++ const uint maxSize = alloc - sizeof(Header);
++
++ IsValidStack stack;
++ stack.push_back(IsValidFrame(root, maxSize));
++ while (!stack.isEmpty()) {
++ const auto [base, size] = stack.last();
++ stack.removeLast();
++ const bool res = base->isObject()
++ ? static_cast<const Object *>(base)->isValidHelper(size, stack)
++ : static_cast<const Array *>(base)->isValidHelper(size, stack);
++ if (!res)
++ return false;
++ }
++ return true;
++ }
++
++ QJsonDocument ConstData::toJsonDocument() const
++ {
++ DocumentStack stack;
++ stack.push_back(DocumentFrame(header->root()));
++
++ while (true) {
++ DocumentFrame &frame = stack.last();
++
++ if (frame.index < frame.base->length()) {
++ const uint i = frame.index++;
++
++ const Value *value = nullptr;
++ QString key;
++ if (frame.isObject) {
++ const Entry *e = static_cast<const Object *>(frame.base)->entryAt(i);
++ value = &e->value;
++ key = e->key();
++ } else {
++ value = static_cast<const Array *>(frame.base)->at(i);
++ }
++
++ const uint type = value->type();
++ if (type == QJsonValue::Array || type == QJsonValue::Object) {
++ const Base *child = value->base(frame.base);
++ // This may reallocate, so frame must not be touched afterwards.
++ stack.push_back(DocumentFrame(child, std::move(key)));
++ continue;
++ }
++
++ const QJsonValue scalar = value->toScalarJsonValue(frame.base);
++ if (frame.isObject)
++ frame.object.insert(key, scalar);
++ else
++ frame.array.append(scalar);
++ continue;
++ }
++
++ // This container is complete: hand it to its parent, or return it.
++ if (stack.size() == 1) {
++ return frame.isObject ? QJsonDocument(std::move(frame.object))
++ : QJsonDocument(std::move(frame.array));
++ } else {
++ const QJsonValue val = frame.isObject ? QJsonValue(std::move(frame.object))
++ : QJsonValue(std::move(frame.array));
++ const QString key = std::move(frame.key);
++ stack.removeLast();
++
++ DocumentFrame &parent = stack.last();
++ if (parent.isObject)
++ parent.object.insert(key, val);
++ else
++ parent.array.append(val);
++ }
++ }
++ }
++
++ uint Base::reserveSpace(uint dataSize, uint posInTable, uint numItems, bool replace)
++ {
++ Q_ASSERT(posInTable <= length());
++ if (size + dataSize >= Value::MaxSize) {
++ qWarning("QJson: Document too large to store in data structure %d %d %d",
++ uint(size), dataSize, Value::MaxSize);
++ return 0;
++ }
++
++ offset off = tableOffset;
++ // move table to new position
++ if (replace) {
++ memmove(reinterpret_cast<char *>(table()) + dataSize, table(), length() * sizeof(offset));
++ } else {
++ memmove(reinterpret_cast<char *>(table() + posInTable + numItems) + dataSize,
++ table() + posInTable, (length() - posInTable) * sizeof(offset));
++ memmove(reinterpret_cast<char *>(table()) + dataSize, table(), posInTable * sizeof(offset));
++ }
++ tableOffset += dataSize;
++ for (uint i = 0; i < numItems; ++i)
++ table()[posInTable + i] = off;
++ size += dataSize;
++ if (!replace) {
++ setLength(length() + numItems);
++ size += numItems * sizeof(offset);
++ }
++ return off;
++ }
++
++ uint Object::indexOf(QStringView key, bool *exists) const
++ {
++ uint min = 0;
++ uint n = length();
++ while (n > 0) {
++ uint half = n >> 1;
++ uint middle = min + half;
++ if (*entryAt(middle) >= key) {
++ n = half;
++ } else {
++ min = middle + 1;
++ n -= half + 1;
++ }
++ }
++ if (min < length() && *entryAt(min) == key) {
++ *exists = true;
++ return min;
++ }
++ *exists = false;
++ return min;
++ }
++
++ bool Object::isValidHelper(uint maxSize, IsValidStack &stack) const
++ {
++ if (!isAlignedOffset(tableOffset))
++ return false;
++
++ if (size > maxSize || tableOffset + length() * sizeof(offset) > size)
++ return false;
++
++ QString lastKey;
++ for (uint i = 0; i < length(); ++i) {
++ const uint entryOffset = table()[i];
++ if (!isAlignedOffset(entryOffset) || entryOffset + sizeof(Entry) >= tableOffset)
++ return false;
++ const Entry *e = entryAt(i);
++ if (!e->isValid(tableOffset - entryOffset))
++ return false;
++ const QString key = e->key();
++ if (key < lastKey)
++ return false;
++ if (!e->value.isValidHelper(this, stack))
++ return false;
++ lastKey = key;
++ }
++ return true;
++ }
++
++ bool Array::isValidHelper(uint maxSize, IsValidStack &stack) const
++ {
++ if (!isAlignedOffset(tableOffset))
++ return false;
++
++ if (size > maxSize || tableOffset + length() * sizeof(offset) > size)
++ return false;
++
++ const offset *values = table();
++ for (uint i = 0; i < length(); ++i) {
++ if (!reinterpret_cast<const Value *>(values + i)->isValidHelper(this, stack))
++ return false;
++ }
++ return true;
++ }
++
++ uint Value::usedStorage(const Base *b) const
++ {
++ uint s = 0;
++ switch (type()) {
++ case QJsonValue::Double:
++ if (!isLatinOrIntValue())
++ s = sizeof(double);
++ break;
++ case QJsonValue::String: {
++ const char *d = data(b);
++ s = isLatinOrIntValue()
++ ? (sizeof(ushort)
++ + qFromLittleEndian(*reinterpret_cast<const ushort *>(d)))
++ : (sizeof(int)
++ + sizeof(ushort) * qFromLittleEndian(*reinterpret_cast<const int *>(d)));
++ break;
++ }
++ case QJsonValue::Array:
++ case QJsonValue::Object:
++ s = base(b)->size;
++ break;
++ case QJsonValue::Null:
++ case QJsonValue::Bool:
++ default:
++ break;
++ }
++ return alignedSize(s);
++ }
++
++ QJsonValue Value::toScalarJsonValue(const Base *b) const
++ {
++ Q_ASSERT(type() != QJsonValue::Array && type() != QJsonValue::Object);
++
++ switch (type()) {
++ case QJsonValue::Null:
++ return QJsonValue(QJsonValue::Null);
++ case QJsonValue::Bool:
++ return QJsonValue(toBoolean());
++ case QJsonValue::Double:
++ return QJsonValue(toDouble(b));
++ case QJsonValue::String:
++ return QJsonValue(toString(b));
++ default:
++ // This handled Undefined, and the out-of-range type codes 6 and 7,
++ // which fit the three bit type field.
++ // We should handle them gracefully, rather than trigger Q_UNREACHABLE()
++ return QJsonValue(QJsonValue::Undefined);
++ }
++ }
++
++ inline bool isValidValueOffset(uint offset, uint tableOffset)
++ {
++ return offset >= sizeof(Base)
++ && isAlignedOffset(offset)
++ && offset + sizeof(uint) <= tableOffset;
++ }
++
++ bool Value::isValidHelper(const Base *b, IsValidStack &stack) const
++ {
++ switch (type()) {
++ case QJsonValue::Null:
++ case QJsonValue::Bool:
++ return true;
++ case QJsonValue::Double:
++ return isLatinOrIntValue() || isValidValueOffset(value(), b->tableOffset);
++ case QJsonValue::String:
++ if (!isValidValueOffset(value(), b->tableOffset))
++ return false;
++ if (isLatinOrIntValue())
++ return asLatin1String(b).isValid(b->tableOffset - value());
++ return asString(b).isValid(b->tableOffset - value());
++ case QJsonValue::Array:
++ case QJsonValue::Object:
++ // do not recurse, instead only check the offset and push the nested
++ // object check on the stack
++ if (isValidValueOffset(value(), b->tableOffset)) {
++ stack.push_back(IsValidFrame(base(b), b->tableOffset - value()));
++ return true;
++ } else {
++ return false;
++ }
++ default:
++ return false;
++ }
++ }
++
++ uint Value::requiredStorage(const QBinaryJsonValue &v, bool *compressed)
++ {
++ *compressed = false;
++ switch (v.type()) {
++ case QJsonValue::Double:
++ if (QBinaryJsonPrivate::compressedNumber(v.toDouble()) != INT_MAX) {
++ *compressed = true;
++ return 0;
++ }
++ return sizeof(double);
++ case QJsonValue::String: {
++ QString s = v.toString();
++ *compressed = QBinaryJsonPrivate::useCompressed(s);
++ return QBinaryJsonPrivate::qStringSize(s, *compressed);
++ }
++ case QJsonValue::Array:
++ case QJsonValue::Object:
++ return v.base ? uint(v.base->size) : sizeof(QBinaryJsonPrivate::Base);
++ case QJsonValue::Undefined:
++ case QJsonValue::Null:
++ case QJsonValue::Bool:
++ break;
++ }
++ return 0;
++ }
++
++ uint Value::valueToStore(const QBinaryJsonValue &v, uint offset)
++ {
++ switch (v.type()) {
++ case QJsonValue::Undefined:
++ case QJsonValue::Null:
++ break;
++ case QJsonValue::Bool:
++ return v.toBool();
++ case QJsonValue::Double: {
++ int c = QBinaryJsonPrivate::compressedNumber(v.toDouble());
++ if (c != INT_MAX)
++ return c;
++ }
++ Q_FALLTHROUGH();
++ case QJsonValue::String:
++ case QJsonValue::Array:
++ case QJsonValue::Object:
++ return offset;
++ }
++ return 0;
++ }
++
++ void Value::copyData(const QBinaryJsonValue &v, char *dest, bool compressed)
++ {
++ switch (v.type()) {
++ case QJsonValue::Double:
++ if (!compressed)
++ qToLittleEndian(v.toDouble(), dest);
++ break;
++ case QJsonValue::String: {
++ const QString str = v.toString();
++ QBinaryJsonPrivate::copyString(dest, str, compressed);
++ break;
++ }
++ case QJsonValue::Array:
++ case QJsonValue::Object: {
++ const QBinaryJsonPrivate::Base *b = v.base;
++ if (!b)
++ b = (v.type() == QJsonValue::Array ? &emptyArray : &emptyObject);
++ memcpy(dest, b, b->size);
++ break;
++ }
++ default:
++ break;
++ }
++ }
++
++ } // namespace QBinaryJsonPrivate
++
++ QT_END_NAMESPACE