Revision control
++ // Copyright (C) 2021 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++ // Qt-Security score:critical reason:low-level-memory-management
++
++ #include "PageAllocation.h"
++ #include "PageReservation.h"
++
++ #include <private/qnumeric_p.h>
++ #include <private/qv4alloca_p.h>
++ #include <private/qv4engine_p.h>
++ #include <private/qv4identifiertable_p.h>
++ #include <private/qv4mapobject_p.h>
++ #include <private/qv4mm_p.h>
++ #include <private/qv4object_p.h>
++ #include <private/qv4profiling_p.h>
++ #include <private/qv4qobjectwrapper_p.h>
++ #include <private/qv4setobject_p.h>
++ #include <private/qv4stackframe_p.h>
++
++ #include <QtQml/qqmlengine.h>
++
++ #include <QtCore/qalgorithms.h>
++ #include <QtCore/qelapsedtimer.h>
++ #include <QtCore/qloggingcategory.h>
++ #include <QtCore/qmap.h>
++ #include <QtCore/qscopedvaluerollback.h>
++
++ #include <algorithm>
++ #include <chrono>
++ #include <cstdlib>
++
++ //#define MM_STATS
++
++ #if !defined(MM_STATS) && !defined(QT_NO_DEBUG)
++ #define MM_STATS
++ #endif
++
++ #if MM_DEBUG
++ #define DEBUG qDebug() << "MM:"
++ #else
++ #define DEBUG if (1) ; else qDebug() << "MM:"
++ #endif
++
++ #ifdef V4_USE_VALGRIND
++ #include <valgrind/valgrind.h>
++ #include <valgrind/memcheck.h>
++ #endif
++
++ #ifdef V4_USE_HEAPTRACK
++ #include <heaptrack_api.h>
++ #endif
++
++ #if OS(QNX)
++ #include <sys/storage.h> // __tls()
++ #endif
++
++ #if USE(PTHREADS) && HAVE(PTHREAD_NP_H)
++ #include <pthread_np.h>
++ #endif
++
++ Q_STATIC_LOGGING_CATEGORY(lcGcStats, "qt.qml.gc.statistics")
++ Q_STATIC_LOGGING_CATEGORY(lcGcAllocatorStats, "qt.qml.gc.allocatorStats")
++ Q_STATIC_LOGGING_CATEGORY(lcGcStateTransitions, "qt.qml.gc.stateTransitions")
++ Q_STATIC_LOGGING_CATEGORY(lcGcForcedRuns, "qt.qml.gc.forcedRuns")
++ Q_STATIC_LOGGING_CATEGORY(lcGcStepExecution, "qt.qml.gc.stepExecution")
++
++ using namespace WTF;
++
++ QT_BEGIN_NAMESPACE
++
++ namespace QV4 {
++
++ enum {
++ MinSlotsGCLimit = QV4::Chunk::AvailableSlots*16,
++ GCOverallocation = 200 /* Max overallocation by the GC in % */
++ };
++
++ struct MemorySegment {
++ enum {
++ #ifdef Q_OS_RTEMS
++ NumChunks = sizeof(quint64),
++ #else
++ NumChunks = 8*sizeof(quint64),
++ #endif
++ SegmentSize = NumChunks*Chunk::ChunkSize,
++ };
++
++ MemorySegment(size_t size)
++ {
++ size += Chunk::ChunkSize; // make sure we can get enough 64k alignment memory
++ if (size < SegmentSize)
++ size = SegmentSize;
++
++ pageReservation = PageReservation::reserve(size, OSAllocator::JSGCHeapPages);
++ base = reinterpret_cast<Chunk *>((reinterpret_cast<quintptr>(pageReservation.base()) + Chunk::ChunkSize - 1) & ~(Chunk::ChunkSize - 1));
++ nChunks = NumChunks;
++ availableBytes = size - (reinterpret_cast<quintptr>(base) - reinterpret_cast<quintptr>(pageReservation.base()));
++ if (availableBytes < SegmentSize)
++ --nChunks;
++ }
++ MemorySegment(MemorySegment &&other) {
++ qSwap(pageReservation, other.pageReservation);
++ qSwap(base, other.base);
++ qSwap(allocatedMap, other.allocatedMap);
++ qSwap(availableBytes, other.availableBytes);
++ qSwap(nChunks, other.nChunks);
++ }
++
++ ~MemorySegment() {
++ if (base)
++ pageReservation.deallocate();
++ }
++
++ void setBit(size_t index) {
++ Q_ASSERT(index < nChunks);
++ quint64 bit = static_cast<quint64>(1) << index;
++ allocatedMap |= bit;
++ }
++ void clearBit(size_t index) {
++ Q_ASSERT(index < nChunks);
++ quint64 bit = static_cast<quint64>(1) << index;
++ allocatedMap &= ~bit;
++ }
++ bool testBit(size_t index) const {
++ Q_ASSERT(index < nChunks);
++ quint64 bit = static_cast<quint64>(1) << index;
++ return (allocatedMap & bit);
++ }
++
++ Chunk *allocate(size_t size);
++ void free(Chunk *chunk, size_t size) {
++ DEBUG << "freeing chunk" << chunk;
++ size_t index = static_cast<size_t>(chunk - base);
++ size_t end = qMin(static_cast<size_t>(NumChunks), index + (size - 1)/Chunk::ChunkSize + 1);
++ while (index < end) {
++ Q_ASSERT(testBit(index));
++ clearBit(index);
++ ++index;
++ }
++
++ size_t pageSize = WTF::pageSize();
++ size = (size + pageSize - 1) & ~(pageSize - 1);
++ #if !defined(Q_OS_LINUX) && !defined(Q_OS_WIN)
++ // Linux and Windows zero out pages that have been decommitted and get committed again.
++ // unfortunately that's not true on other OSes (e.g. BSD based ones), so zero out the
++ // memory before decommit, so that we can be sure that all chunks we allocate will be
++ // zero initialized.
++ memset(chunk, 0, size);
++ #endif
++ pageReservation.decommit(chunk, size);
++ }
++
++ bool contains(Chunk *c) const {
++ return c >= base && c < base + nChunks;
++ }
++
++ PageReservation pageReservation;
++ Chunk *base = nullptr;
++ quint64 allocatedMap = 0;
++ size_t availableBytes = 0;
++ uint nChunks = 0;
++ };
++
++ Chunk *MemorySegment::allocate(size_t size)
++ {
++ if (!allocatedMap && size >= SegmentSize) {
++ // chunk allocated for one huge allocation
++ Q_ASSERT(availableBytes >= size);
++ pageReservation.commit(base, size);
++ allocatedMap = ~static_cast<quint64>(0);
++ return base;
++ }
++ size_t requiredChunks = (size + sizeof(Chunk) - 1)/sizeof(Chunk);
++ uint sequence = 0;
++ Chunk *candidate = nullptr;
++ for (uint i = 0; i < nChunks; ++i) {
++ if (!testBit(i)) {
++ if (!candidate)
++ candidate = base + i;
++ ++sequence;
++ } else {
++ candidate = nullptr;
++ sequence = 0;
++ }
++ if (sequence == requiredChunks) {
++ pageReservation.commit(candidate, size);
++ for (uint i = 0; i < requiredChunks; ++i)
++ setBit(candidate - base + i);
++ DEBUG << "allocated chunk " << candidate << Qt::hex << size;
++
++ return candidate;
++ }
++ }
++ return nullptr;
++ }
++
++ struct ChunkAllocator {
++ ChunkAllocator() {}
++
++ size_t requiredChunkSize(size_t size) {
++ size += Chunk::HeaderSize; // space required for the Chunk header
++ size_t pageSize = WTF::pageSize();
++ size = (size + pageSize - 1) & ~(pageSize - 1); // align to page sizes
++ if (size < Chunk::ChunkSize)
++ size = Chunk::ChunkSize;
++ return size;
++ }
++
++ Chunk *allocate(size_t size = 0);
++ void free(Chunk *chunk, size_t size = 0);
++
++ std::vector<MemorySegment> memorySegments;
++ };
++
++ Chunk *ChunkAllocator::allocate(size_t size)
++ {
++ size = requiredChunkSize(size);
++ for (auto &m : memorySegments) {
++ if (~m.allocatedMap) {
++ Chunk *c = m.allocate(size);
++ if (c)
++ return c;
++ }
++ }
++
++ // allocate a new segment
++ memorySegments.push_back(MemorySegment(size));
++ Chunk *c = memorySegments.back().allocate(size);
++ Q_ASSERT(c);
++ return c;
++ }
++
++ void ChunkAllocator::free(Chunk *chunk, size_t size)
++ {
++ size = requiredChunkSize(size);
++ for (auto &m : memorySegments) {
++ if (m.contains(chunk)) {
++ m.free(chunk, size);
++ return;
++ }
++ }
++ Q_ASSERT(false);
++ }
++
++ #ifdef DUMP_SWEEP
++ QString binary(quintptr n) {
++ QString s = QString::number(n, 2);
++ while (s.length() < 64)
++ s.prepend(QChar::fromLatin1('0'));
++ return s;
++ }
++ #define SDUMP qDebug
++ #else
++ QString binary(quintptr) { return QString(); }
++ #define SDUMP if (1) ; else qDebug
++ #endif
++
++ bool Chunk::sweep(ExecutionEngine *engine)
++ {
++ bool hasUsedSlots = false;
++ SDUMP() << "sweeping chunk" << this;
++ HeapItem *o = realBase();
++ bool lastSlotFree = false;
++ for (uint i = 0; i < Chunk::EntriesInBitmap; ++i) {
++ // Snapshots. The destructors below may allocate, and setAllocatedSlots() then
++ // sets object and extends bits in this very word. We have to remember what we
++ // decided to free, so that we can clear exactly that further down rather than
++ // assigning the words wholesale and dropping those bits again.
++ const quintptr objectsToFree = objectBitmap[i] ^ blackBitmap[i];
++ const quintptr extendsBefore = extendsBitmap[i];
++
++ quintptr toFree = objectsToFree;
++ Q_ASSERT((toFree & objectBitmap[i]) == toFree); // check all black objects are marked as being used
++ quintptr e = extendsBefore;
++ SDUMP() << " index=" << i;
++ SDUMP() << " toFree =" << binary(toFree);
++ SDUMP() << " black =" << binary(blackBitmap[i]);
++ SDUMP() << " object =" << binary(objectBitmap[i]);
++ SDUMP() << " extends =" << binary(e);
++ if (lastSlotFree)
++ e &= (e + 1); // clear all lowest extent bits
++ while (toFree) {
++ uint index = qCountTrailingZeroBits(toFree);
++ quintptr bit = (static_cast<quintptr>(1) << index);
++
++ toFree ^= bit; // mask out freed slot
++
++ // remove all extends slots that have been freed
++ // this is a bit of bit trickery.
++ quintptr mask = (bit << 1) - 1; // create a mask of 1's to the right of and up to the current bit
++ quintptr objmask = e | mask; // or'ing mask with e gives all ones until the end of the current object
++ quintptr result = objmask + 1;
++ Q_ASSERT(qCountTrailingZeroBits(result) - index != 0); // ensure we freed something
++ result |= mask; // ensure we don't clear stuff to the right of the current object
++ e &= result;
++
++ HeapItem *itemToFree = o + index;
++ Heap::Base *b = *itemToFree;
++ const VTable *v = b->internalClass->vtable;
++ // if (Q_UNLIKELY(classCountPtr))
++ // classCountPtr(v->className);
++ if (v->destroy)
++ v->destroy(b);
++ #ifdef V4_USE_HEAPTRACK
++ heaptrack_report_free(itemToFree);
++ #endif
++ }
++ const quintptr extendsToFree = extendsBefore & ~e;
++ Q_V4_PROFILE_DEALLOC(engine,
++ qPopulationCount(objectsToFree | extendsToFree) * Chunk::SlotSize,
++ Profiling::RegularItem);
++ objectBitmap[i] &= ~objectsToFree;
++ extendsBitmap[i] &= ~extendsToFree;
++ hasUsedSlots |= (objectBitmap[i] != 0);
++ lastSlotFree = !((objectBitmap[i]|extendsBitmap[i]) >> (sizeof(quintptr)*8 - 1));
++ SDUMP() << " new extends =" << binary(e);
++ SDUMP() << " lastSlotFree" << lastSlotFree;
++ Q_ASSERT((objectBitmap[i] & extendsBitmap[i]) == 0);
++ o += Chunk::Bits;
++ }
++ return hasUsedSlots;
++ }
++
++ void Chunk::freeAll(ExecutionEngine *engine)
++ {
++ HeapItem *o = realBase();
++ for (uint i = 0; i < Chunk::EntriesInBitmap; ++i) {
++ quintptr toFree = objectBitmap[i];
++ quintptr e = extendsBitmap[i];
++ while (toFree) {
++ uint index = qCountTrailingZeroBits(toFree);
++ quintptr bit = (static_cast<quintptr>(1) << index);
++
++ toFree ^= bit; // mask out freed slot
++
++ // remove all extends slots that have been freed
++ // this is a bit of bit trickery.
++ quintptr mask = (bit << 1) - 1; // create a mask of 1's to the right of and up to the current bit
++ quintptr objmask = e | mask; // or'ing mask with e gives all ones until the end of the current object
++ quintptr result = objmask + 1;
++ Q_ASSERT(qCountTrailingZeroBits(result) - index != 0); // ensure we freed something
++ result |= mask; // ensure we don't clear stuff to the right of the current object
++ e &= result;
++
++ HeapItem *itemToFree = o + index;
++ Heap::Base *b = *itemToFree;
++ if (b->internalClass->vtable->destroy)
++ b->internalClass->vtable->destroy(b);
++ #ifdef V4_USE_HEAPTRACK
++ heaptrack_report_free(itemToFree);
++ #endif
++ }
++ Q_V4_PROFILE_DEALLOC(engine, (qPopulationCount(objectBitmap[i]|extendsBitmap[i])
++ - qPopulationCount(e)) * Chunk::SlotSize, Profiling::RegularItem);
++ objectBitmap[i] = 0;
++ extendsBitmap[i] = e;
++ o += Chunk::Bits;
++ }
++ }
++
++ void Chunk::resetBlackBits()
++ {
++ memset(blackBitmap, 0, sizeof(blackBitmap));
++ }
++
++ void Chunk::sortIntoBins(HeapItem **bins, uint nBins)
++ {
++ HeapItem *base = realBase();
++ #if QT_POINTER_SIZE == 8
++ const int start = 0;
++ #else
++ const int start = 1;
++ #endif
++ uint freeSlots = 0;
++ uint allocatedSlots = 0;
++
++ for (int i = start; i < EntriesInBitmap; ++i) {
++ quintptr usedSlots = (objectBitmap[i]|extendsBitmap[i]);
++ #if QT_POINTER_SIZE == 8
++ if (!i)
++ usedSlots |= (static_cast<quintptr>(1) << (HeaderSize/SlotSize)) - 1;
++ #endif
++ allocatedSlots += qPopulationCount(usedSlots);
++ while (1) {
++ uint index = qCountTrailingZeroBits(usedSlots + 1);
++ if (index == Bits)
++ break;
++ uint freeStart = i*Bits + index;
++ usedSlots &= ~((static_cast<quintptr>(1) << index) - 1);
++ while (!usedSlots) {
++ if (++i < EntriesInBitmap) {
++ usedSlots = (objectBitmap[i]|extendsBitmap[i]);
++ } else {
++ Q_ASSERT(i == EntriesInBitmap);
++ // Overflows to 0 when counting trailing zeroes above in next iteration.
++ // Then, all the bits are zeroes and we break.
++ usedSlots = std::numeric_limits<quintptr>::max();
++ break;
++ }
++ allocatedSlots += qPopulationCount(usedSlots);
++ }
++ HeapItem *freeItem = base + freeStart;
++
++ index = qCountTrailingZeroBits(usedSlots);
++ usedSlots |= (quintptr(1) << index) - 1;
++ uint freeEnd = i*Bits + index;
++ uint nSlots = freeEnd - freeStart;
++ freeSlots += nSlots;
++ Q_ASSERT(freeEnd > freeStart && freeEnd <= NumSlots);
++ freeItem->freeData.availableSlots = nSlots;
++ uint bin = qMin(nBins - 1, nSlots);
++ freeItem->freeData.next = bins[bin];
++ bins[bin] = freeItem;
++ }
++ }
++ Q_ASSERT(freeSlots + allocatedSlots == (EntriesInBitmap - start) * 8 * sizeof(quintptr));
++ }
++
++ HeapItem *BlockAllocator::allocate(size_t size, bool forceAllocation) {
++ Q_ASSERT((size % Chunk::SlotSize) == 0);
++ size_t slotsRequired = size >> Chunk::SlotSizeShift;
++
++ if (allocationStats)
++ ++allocationStats[binForSlots(slotsRequired)];
++
++ HeapItem **last;
++
++ HeapItem *m;
++
++ if (slotsRequired < NumBins - 1) {
++ m = freeBins[slotsRequired];
++ if (m) {
++ freeBins[slotsRequired] = m->freeData.next;
++ goto done;
++ }
++ }
++
++ if (nFree >= slotsRequired) {
++ // use bump allocation
++ Q_ASSERT(nextFree);
++ m = nextFree;
++ nextFree += slotsRequired;
++ nFree -= slotsRequired;
++ goto done;
++ }
++
++ // search last bin for a large enough item
++ last = &freeBins[NumBins - 1];
++ while ((m = *last)) {
++ if (m->freeData.availableSlots >= slotsRequired) {
++ *last = m->freeData.next; // take it out of the list
++
++ size_t remainingSlots = m->freeData.availableSlots - slotsRequired;
++ if (remainingSlots == 0)
++ goto done;
++
++ HeapItem *remainder = m + slotsRequired;
++ if (remainingSlots > nFree) {
++ if (nFree) {
++ size_t bin = binForSlots(nFree);
++ nextFree->freeData.next = freeBins[bin];
++ nextFree->freeData.availableSlots = nFree;
++ freeBins[bin] = nextFree;
++ }
++ nextFree = remainder;
++ nFree = remainingSlots;
++ } else {
++ remainder->freeData.availableSlots = remainingSlots;
++ size_t binForRemainder = binForSlots(remainingSlots);
++ remainder->freeData.next = freeBins[binForRemainder];
++ freeBins[binForRemainder] = remainder;
++ }
++ goto done;
++ }
++ last = &m->freeData.next;
++ }
++
++ if (slotsRequired < NumBins - 1) {
++ // check if we can split up another slot
++ for (size_t i = slotsRequired + 1; i < NumBins - 1; ++i) {
++ m = freeBins[i];
++ if (m) {
++ freeBins[i] = m->freeData.next; // take it out of the list
++ size_t remainingSlots = i - slotsRequired;
++ Q_ASSERT(remainingSlots < NumBins - 1);
++ HeapItem *remainder = m + slotsRequired;
++ remainder->freeData.availableSlots = remainingSlots;
++ remainder->freeData.next = freeBins[remainingSlots];
++ freeBins[remainingSlots] = remainder;
++ goto done;
++ }
++ }
++ }
++
++ if (!m) {
++ if (!forceAllocation)
++ return nullptr;
++ if (nFree) {
++ // Save any remaining slots of the current chunk
++ // for later, smaller allocations.
++ size_t bin = binForSlots(nFree);
++ nextFree->freeData.next = freeBins[bin];
++ nextFree->freeData.availableSlots = nFree;
++ freeBins[bin] = nextFree;
++ }
++ Chunk *newChunk = chunkAllocator->allocate();
++ Q_V4_PROFILE_ALLOC(engine, Chunk::DataSize, Profiling::HeapPage);
++ chunks.push_back(newChunk);
++ nextFree = newChunk->first();
++ nFree = Chunk::AvailableSlots;
++ m = nextFree;
++ nextFree += slotsRequired;
++ nFree -= slotsRequired;
++ }
++
++ done:
++ m->setAllocatedSlots(slotsRequired);
++ Q_V4_PROFILE_ALLOC(engine, slotsRequired * Chunk::SlotSize, Profiling::RegularItem);
++ #ifdef V4_USE_HEAPTRACK
++ heaptrack_report_alloc(m, slotsRequired * Chunk::SlotSize);
++ #endif
++ return m;
++ }
++
++ void BlockAllocator::sweep()
++ {
++ const auto firstEmptyChunkPos = partition(chunks, [this](const std::size_t i) {
++ return chunks.at(i)->sweep(engine);
++ });
++ const auto firstEmptyChunk = chunks.begin() + firstEmptyChunkPos;
++
++ nextFree = nullptr;
++ nFree = 0;
++ memset(freeBins, 0, sizeof(freeBins));
++
++ usedSlotsAfterLastSweep = 0;
++
++ std::for_each(chunks.begin(), firstEmptyChunk, [this](Chunk *c) {
++ c->sortIntoBins(freeBins, NumBins);
++ usedSlotsAfterLastSweep += c->nUsedSlots();
++ });
++
++ // only free the chunks at the end to avoid that the sweep() calls indirectly
++ // access freed memory
++ std::for_each(firstEmptyChunk, chunks.end(), [this](Chunk *c) {
++ Q_V4_PROFILE_DEALLOC(engine, Chunk::DataSize, Profiling::HeapPage);
++ chunkAllocator->free(c);
++ });
++
++ chunks.erase(firstEmptyChunk, chunks.end());
++ }
++
++ void BlockAllocator::freeAll()
++ {
++ for (auto c : chunks)
++ c->freeAll(engine);
++ for (auto c : chunks) {
++ Q_V4_PROFILE_DEALLOC(engine, Chunk::DataSize, Profiling::HeapPage);
++ chunkAllocator->free(c);
++ }
++ }
++
++ void BlockAllocator::resetBlackBits()
++ {
++ for (auto c : chunks)
++ c->resetBlackBits();
++ }
++
++ HeapItem *HugeItemAllocator::allocate(size_t size) {
++ MemorySegment *m = nullptr;
++ Chunk *c = nullptr;
++ if (size >= MemorySegment::SegmentSize/2) {
++ // too large to handle through the ChunkAllocator, let's get our own memory segement
++ size += Chunk::HeaderSize; // space required for the Chunk header
++ size_t pageSize = WTF::pageSize();
++ size = (size + pageSize - 1) & ~(pageSize - 1); // align to page sizes
++ m = new MemorySegment(size);
++ c = m->allocate(size);
++ } else {
++ c = chunkAllocator->allocate(size);
++ }
++ Q_ASSERT(c);
++ chunks.push_back(HugeChunk{m, c, size});
++ Chunk::setBit(c->objectBitmap, c->first() - c->realBase());
++ Q_V4_PROFILE_ALLOC(engine, size, Profiling::LargeItem);
++ #ifdef V4_USE_HEAPTRACK
++ heaptrack_report_alloc(c, size);
++ #endif
++ return c->first();
++ }
++
++ static void freeHugeChunk(ChunkAllocator *chunkAllocator, const HugeItemAllocator::HugeChunk &c)
++ {
++ HeapItem *itemToFree = c.chunk->first();
++ Heap::Base *b = *itemToFree;
++ const VTable *v = b->internalClass->vtable;
++
++ if (v->destroy)
++ v->destroy(b);
++ if (c.segment) {
++ // own memory segment
++ c.segment->free(c.chunk, c.size);
++ delete c.segment;
++ } else {
++ chunkAllocator->free(c.chunk, c.size);
++ }
++ #ifdef V4_USE_HEAPTRACK
++ heaptrack_report_free(c.chunk);
++ #endif
++ }
++
++ void HugeItemAllocator::sweep()
++ {
++ auto isBlack = [this] (const HugeChunk &c) {
++ bool b = c.chunk->first()->isBlack();
++ Chunk::clearBit(c.chunk->blackBitmap, c.chunk->first() - c.chunk->realBase());
++ if (!b) {
++ Q_V4_PROFILE_DEALLOC(engine, c.size, Profiling::LargeItem);
++ freeHugeChunk(chunkAllocator, c);
++ }
++ return !b;
++ };
++
++ auto newEnd = std::remove_if(chunks.begin(), chunks.end(), isBlack);
++ chunks.erase(newEnd, chunks.end());
++ }
++
++ void HugeItemAllocator::resetBlackBits()
++ {
++ for (auto c : chunks)
++ Chunk::clearBit(c.chunk->blackBitmap, c.chunk->first() - c.chunk->realBase());
++ }
++
++ void HugeItemAllocator::freeAll()
++ {
++ for (auto &c : chunks) {
++ Q_V4_PROFILE_DEALLOC(engine, c.size, Profiling::LargeItem);
++ freeHugeChunk(chunkAllocator, c);
++ }
++ }
++
++ namespace {
++ using ExtraData = GCStateInfo::ExtraData;
++ GCState markStart(GCStateMachine *that, ExtraData &)
++ {
++ //Initialize the mark stack
++ that->mm->m_markStack = std::make_unique<MarkStack>(that->mm->engine);
++ that->mm->engine->isGCOngoing = true;
++ return GCState::MarkGlobalObject;
++ }
++
++ GCState markGlobalObject(GCStateMachine *that, ExtraData &)
++ {
++ that->mm->engine->markObjects(that->mm->m_markStack.get());
++ return GCState::MarkJSStack;
++ }
++
++ GCState markJSStack(GCStateMachine *that, ExtraData &)
++ {
++ that->mm->collectFromJSStack(that->mm->markStack());
++ return GCState::InitMarkPersistentValues;
++ }
++
++ GCState initMarkPersistentValues(GCStateMachine *that, ExtraData &stateData)
++ {
++ if (!that->mm->m_persistentValues)
++ return GCState::InitMarkWeakValues; // no persistent values to mark
++ stateData = GCIteratorStorage { that->mm->m_persistentValues->begin() };
++ return GCState::MarkPersistentValues;
++ }
++
++ enum: int {
++ MarkLoopIterationCount = 1024,
++ MarkLoopIterationCountForDrain = 10240,
++ };
++
++ bool wasDrainNecessary(MarkStack *ms, QDeadlineTimer deadline)
++ {
++ if (ms->remainingBeforeSoftLimit() > MarkLoopIterationCount)
++ return false;
++ // drain
++ ms->drain(deadline);
++ return true;
++ }
++
++ GCState markPersistentValues(GCStateMachine *that, ExtraData &stateData) {
++ auto markStack = that->mm->markStack();
++ if (wasDrainNecessary(markStack, that->deadline) && that->deadline.hasExpired())
++ return GCState::MarkPersistentValues;
++ PersistentValueStorage::Iterator& it = get<GCIteratorStorage>(stateData).it;
++ // avoid repeatedly hitting the timer constantly by batching iterations
++ for (int i = 0; i < MarkLoopIterationCount; ++i) {
++ if (!it.p)
++ return GCState::InitMarkWeakValues;
++ if (Managed *m = (*it).as<Managed>())
++ m->mark(markStack);
++ ++it;
++ }
++ return GCState::MarkPersistentValues;
++ }
++
++ GCState initMarkWeakValues(GCStateMachine *that, ExtraData &stateData)
++ {
++ stateData = GCIteratorStorage { that->mm->m_weakValues->begin() };
++ return GCState::MarkWeakValues;
++ }
++
++ GCState markWeakValues(GCStateMachine *that, ExtraData &stateData)
++ {
++ auto markStack = that->mm->markStack();
++ if (wasDrainNecessary(markStack, that->deadline) && that->deadline.hasExpired())
++ return GCState::MarkWeakValues;
++ PersistentValueStorage::Iterator& it = get<GCIteratorStorage>(stateData).it;
++ // avoid repeatedly hitting the timer constantly by batching iterations
++ for (int i = 0; i < MarkLoopIterationCount; ++i) {
++ if (!it.p)
++ return GCState::MarkDrain;
++ QObjectWrapper *qobjectWrapper = (*it).as<QObjectWrapper>();
++ ++it;
++ if (!qobjectWrapper)
++ continue;
++ QObject *qobject = qobjectWrapper->object();
++ if (!qobject)
++ continue;
++ bool keepAlive = QQmlData::keepAliveDuringGarbageCollection(qobject);
++
++ if (!keepAlive) {
++ if (QObject *parent = qobject->parent()) {
++ while (parent->parent())
++ parent = parent->parent();
++ keepAlive = QQmlData::keepAliveDuringGarbageCollection(parent);
++ }
++ }
++
++ if (keepAlive)
++ qobjectWrapper->mark(that->mm->markStack());
++ }
++ return GCState::MarkWeakValues;
++ }
++
++ GCState markDrain(GCStateMachine *that, ExtraData &)
++ {
++ if (that->deadline.isForever()) {
++ that->mm->markStack()->drain();
++ return GCState::MarkReady;
++ }
++ auto drainState = that->mm->m_markStack->drain(that->deadline);
++ return drainState == MarkStack::DrainState::Complete
++ ? GCState::MarkReady
++ : GCState::MarkDrain;
++ }
++
++ GCState markReady(GCStateMachine *that, ExtraData &)
++ {
++ auto isIncrementalRun = [](GCStateMachine* that){
++ return !that->mm->aggressiveGC && that->timeLimit.count() > 0;
++ };
++
++ if (that->mm->crossValidateIncrementalGC && isIncrementalRun(that))
++ return GCState::CrossValidateIncrementalMarkPhase;
++ return GCState::InitCallDestroyObjects;
++ }
++
++ GCState crossValidateIncrementalMarkPhase(GCStateMachine *that, ExtraData &)
++ {
++ struct {
++ Chunk* operator()(Chunk* chunk) { return chunk; }
++ Chunk* operator()(const HugeItemAllocator::HugeChunk& chunk) { return chunk.chunk; }
++ } getChunk{};
++
++ auto takeBlackBitmap = [&getChunk](auto& allocator, std::vector<quintptr>& storage){
++ for (auto chunk : allocator.chunks) {
++ for (auto& bitmap : getChunk(chunk)->blackBitmap) {
++ storage.push_back(bitmap);
++ }
++ getChunk(chunk)->resetBlackBits();
++ }
++ };
++
++ auto runMarkPhase = [](GCStateMachine* that) {
++ that->reset();
++ that->mm->m_markStack.reset();
++
++ while (that->state != GCStateMachine::MarkReady) {
++ GCStateInfo& stateInfo = that->stateInfoMap[int(that->state)];
++ that->state = stateInfo.execute(that, that->stateData);
++ }
++ };
++
++ auto checkBlackBitmap = [&that, &getChunk](auto& allocator, const std::vector<quintptr>& storedBitmap) {
++ auto reportError = [&allocator, &getChunk, &that](std::size_t chunk_index, std::size_t bitmap_index, uint bit_index){
++ #ifdef QT_BUILD_INTERNAL
++ // If we're collecting errors, don't output the debug message.
++ if (auto errors = that->bitmapErrors) {
++ errors->emplace_back(chunk_index, bitmap_index, bit_index);
++ return;
++ }
++ #endif
++
++ Q_UNUSED(that);
++ auto object = reinterpret_cast<Heap::Base*>(getChunk(allocator.chunks[chunk_index])->realBase() + (bit_index + (bitmap_index*Chunk::Bits)));
++ qDebug() << "Cross Validation Error on chunk" << chunk_index
++ << "on bitmap piece" << bitmap_index << "and bit" << bit_index
++ << ((object->internalClass) ? "With type" : "")
++ << ((object->internalClass) ?
++ Managed::typeToString(Managed::Type(object->internalClass->vtable->type)) : QString());
++ };
++
++ auto original = storedBitmap.begin();
++ for (std::size_t chunk_index = 0; original != storedBitmap.end() && chunk_index < allocator.chunks.size(); ++chunk_index) {
++ for (std::size_t bitmap_index = 0; bitmap_index < Chunk::EntriesInBitmap; ++bitmap_index) {
++ if (auto differences = (~(*original)) & getChunk(allocator.chunks[chunk_index])->blackBitmap[bitmap_index]) {
++ while (differences != 0) {
++ uint bit_index = qCountTrailingZeroBits(differences);
++ reportError(chunk_index, bitmap_index, bit_index);
++ differences ^= quintptr{1} << bit_index;
++ }
++ }
++ ++original;
++ }
++ }
++ };
++
++ #ifdef QT_BUILD_INTERNAL
++ if (auto *errors = that->bitmapErrors)
++ errors->clear();
++ #endif
++
++ std::vector<quintptr> blockBitmap{};
++ blockBitmap.reserve(Chunk::EntriesInBitmap * that->mm->blockAllocator.chunks.size());
++ takeBlackBitmap(that->mm->blockAllocator, blockBitmap);
++
++ std::vector<quintptr> hugeItemBitmap{};
++ hugeItemBitmap.reserve(Chunk::EntriesInBitmap * that->mm->hugeItemAllocator.chunks.size());
++ takeBlackBitmap(that->mm->hugeItemAllocator, hugeItemBitmap);
++
++ std::vector<quintptr> internalClassBitmap{};
++ internalClassBitmap.reserve(Chunk::EntriesInBitmap * that->mm->icAllocator.chunks.size());
++ takeBlackBitmap(that->mm->icAllocator, internalClassBitmap);
++
++ runMarkPhase(that);
++
++ checkBlackBitmap(that->mm->blockAllocator, blockBitmap);
++ checkBlackBitmap(that->mm->hugeItemAllocator, hugeItemBitmap);
++ checkBlackBitmap(that->mm->icAllocator, internalClassBitmap);
++
++ return GCState::InitCallDestroyObjects;
++ }
++
++ /** \!internal
++ collects new references from the stack, then drains the mark stack again
++ */
++ void redrain(GCStateMachine *that)
++ {
++ that->mm->collectFromJSStack(that->mm->markStack());
++ that->mm->m_markStack->drain();
++ }
++
++ GCState initCallDestroyObjects(GCStateMachine *that, ExtraData &stateData)
++ {
++ // as we don't have a deletion barrier, we need to rescan the stack
++ redrain(that);
++ if (!that->mm->m_weakValues)
++ return GCState::FreeWeakMaps; // no need to call destroy objects
++ stateData = GCIteratorStorage { that->mm->m_weakValues->begin() };
++ return GCState::CallDestroyObjects;
++ }
++ GCState callDestroyObject(GCStateMachine *that, ExtraData &stateData)
++ {
++ PersistentValueStorage::Iterator& it = get<GCIteratorStorage>(stateData).it;
++ // destroyObject might call user code, which really shouldn't call back into the gc
++ auto oldState = std::exchange(that->mm->gcBlocked, QV4::MemoryManager::Blockness::InCriticalSection);
++ auto cleanup = qScopeGuard([&]() {
++ that->mm->gcBlocked = oldState;
++ });
++ // avoid repeatedly hitting the timer constantly by batching iterations
++ for (int i = 0; i < MarkLoopIterationCount; ++i) {
++ if (!it.p)
++ return GCState::FreeWeakMaps;
++ Managed *m = (*it).managed();
++ ++it;
++ if (!m || m->markBit())
++ continue;
++ // we need to call destroyObject on qobjectwrappers now, so that they can emit the destroyed
++ // signal before we start sweeping the heap
++ if (QObjectWrapper *qobjectWrapper = m->as<QObjectWrapper>())
++ qobjectWrapper->destroyObject(/*lastSweep =*/false);
++ }
++ return GCState::CallDestroyObjects;
++ }
++
++ void freeWeakMaps(MemoryManager *mm)
++ {
++ for (auto [map, lastMap] = std::tuple {mm->weakMaps, &mm->weakMaps }; map; map = map->nextWeakMap) {
++ if (!map->isMarked())
++ continue;
++ map->removeUnmarkedKeys();
++ *lastMap = map;
++ lastMap = &map->nextWeakMap;
++ }
++ }
++
++ GCState freeWeakMaps(GCStateMachine *that, ExtraData &)
++ {
++ freeWeakMaps(that->mm);
++ return GCState::FreeWeakSets;
++ }
++
++ void freeWeakSets(MemoryManager *mm)
++ {
++ for (auto [set, lastSet] = std::tuple {mm->weakSets, &mm->weakSets}; set; set = set->nextWeakSet) {
++
++ if (!set->isMarked())
++ continue;
++ set->removeUnmarkedKeys();
++ *lastSet = set;
++ lastSet = &set->nextWeakSet;
++ }
++ }
++
++ GCState freeWeakSets(GCStateMachine *that, ExtraData &)
++ {
++ freeWeakSets(that->mm);
++ return GCState::HandleQObjectWrappers;
++ }
++
++ GCState handleQObjectWrappers(GCStateMachine *that, ExtraData &)
++ {
++ that->mm->cleanupDeletedQObjectWrappersInSweep();
++ return GCState::DoSweep;
++ }
++
++ GCState doSweep(GCStateMachine *that, ExtraData &)
++ {
++ auto mm = that->mm;
++
++ mm->engine->identifierTable->sweep();
++ mm->blockAllocator.sweep();
++ mm->hugeItemAllocator.sweep();
++ mm->icAllocator.sweep();
++
++ // reset all black bits
++ mm->blockAllocator.resetBlackBits();
++ mm->hugeItemAllocator.resetBlackBits();
++ mm->icAllocator.resetBlackBits();
++
++ mm->usedSlotsAfterLastFullSweep = mm->blockAllocator.usedSlotsAfterLastSweep + mm->icAllocator.usedSlotsAfterLastSweep;
++ mm->gcBlocked = MemoryManager::Unblocked;
++ mm->m_markStack.reset();
++ mm->engine->isGCOngoing = false;
++
++ mm->updateUnmanagedHeapSizeGCLimit();
++
++ return GCState::Invalid;
++ }
++
++ }
++
++
++ MemoryManager::MemoryManager(ExecutionEngine *engine)
++ : engine(engine)
++ , chunkAllocator(new ChunkAllocator)
++ , blockAllocator(chunkAllocator, engine)
++ , icAllocator(chunkAllocator, engine)
++ , hugeItemAllocator(chunkAllocator, engine)
++ , m_persistentValues(new PersistentValueStorage(engine))
++ , m_weakValues(new PersistentValueStorage(engine))
++ , unmanagedHeapSizeGCLimit(MinUnmanagedHeapSizeGCLimit)
++ , aggressiveGC(!qEnvironmentVariableIsEmpty("QV4_MM_AGGRESSIVE_GC"))
++ , crossValidateIncrementalGC(qEnvironmentVariableIsSet("QV4_MM_CROSS_VALIDATE_INCREMENTAL_GC"))
++ , statistics(lcGcStats().isDebugEnabled() ? std::make_unique<Statistics>() : std::unique_ptr<Statistics>())
++ , collectorStatistics(lcGcAllocatorStats().isDebugEnabled()
++ ? std::make_unique<CollectorStatistics>()
++ : std::unique_ptr<CollectorStatistics>())
++ {
++ #ifdef V4_USE_VALGRIND
++ VALGRIND_CREATE_MEMPOOL(this, 0, true);
++ #endif
++ if (statistics) {
++ memset(statistics->allocations, 0, sizeof(statistics->allocations));
++ blockAllocator.allocationStats = statistics->allocations;
++ }
++
++ gcStateMachine = std::make_unique<GCStateMachine>();
++ gcStateMachine->mm = this;
++
++ gcStateMachine->stateInfoMap[GCState::MarkStart] = {
++ markStart,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::MarkGlobalObject] = {
++ markGlobalObject,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::MarkJSStack] = {
++ markJSStack,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::InitMarkPersistentValues] = {
++ initMarkPersistentValues,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::MarkPersistentValues] = {
++ markPersistentValues,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::InitMarkWeakValues] = {
++ initMarkWeakValues,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::MarkWeakValues] = {
++ markWeakValues,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::MarkDrain] = {
++ markDrain,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::MarkReady] = {
++ markReady,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::CrossValidateIncrementalMarkPhase] = {
++ crossValidateIncrementalMarkPhase,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::InitCallDestroyObjects] = {
++ initCallDestroyObjects,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::CallDestroyObjects] = {
++ callDestroyObject,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::FreeWeakMaps] = {
++ freeWeakMaps,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::FreeWeakSets] = {
++ freeWeakSets,
++ true, // ensure that handleQObjectWrappers runs in isolation
++ };
++ gcStateMachine->stateInfoMap[GCState::HandleQObjectWrappers] = {
++ handleQObjectWrappers,
++ false,
++ };
++ gcStateMachine->stateInfoMap[GCState::DoSweep] = {
++ doSweep,
++ false,
++ };
++ }
++
++ Heap::Base *MemoryManager::allocString(std::size_t unmanagedSize)
++ {
++ const size_t stringSize = align(sizeof(Heap::String));
++ #ifdef MM_STATS
++ lastAllocRequestedSlots = stringSize >> Chunk::SlotSizeShift;
++ ++allocationCount;
++ #endif
++ unmanagedHeapSize += unmanagedSize;
++
++ HeapItem *m = allocate(&blockAllocator, stringSize);
++ memset(m, 0, stringSize);
++ return *m;
++ }
++
++ Heap::Base *MemoryManager::allocData(std::size_t size)
++ {
++ #ifdef MM_STATS
++ lastAllocRequestedSlots = size >> Chunk::SlotSizeShift;
++ ++allocationCount;
++ #endif
++
++ Q_ASSERT(size >= Chunk::SlotSize);
++ Q_ASSERT(size % Chunk::SlotSize == 0);
++
++ HeapItem *m = allocate(&blockAllocator, size);
++ memset(m, 0, size);
++ return *m;
++ }
++
++ Heap::Object *MemoryManager::allocObjectWithMemberData(const QV4::VTable *vtable, uint nMembers)
++ {
++ uint size = (vtable->nInlineProperties + vtable->inlinePropertyOffset)*sizeof(Value);
++ Q_ASSERT(!(size % sizeof(HeapItem)));
++
++ Heap::Object *o;
++ if (nMembers <= vtable->nInlineProperties) {
++ o = static_cast<Heap::Object *>(allocData(size));
++ } else {
++ // Allocate both in one go through the block allocator
++ nMembers -= vtable->nInlineProperties;
++ std::size_t memberSize = align(sizeof(Heap::MemberData) + (nMembers - 1)*sizeof(Value));
++ size_t totalSize = size + memberSize;
++ Heap::MemberData *m;
++ if (totalSize > Chunk::DataSize) {
++ o = static_cast<Heap::Object *>(allocData(size));
++ m = hugeItemAllocator.allocate(memberSize)->as<Heap::MemberData>();
++ } else {
++ HeapItem *mh = reinterpret_cast<HeapItem *>(allocData(totalSize));
++ Heap::Base *b = *mh;
++ o = static_cast<Heap::Object *>(b);
++ mh += (size >> Chunk::SlotSizeShift);
++ m = mh->as<Heap::MemberData>();
++ Chunk *c = mh->chunk();
++ size_t index = mh - c->realBase();
++ Chunk::setBit(c->objectBitmap, index);
++ Chunk::clearBit(c->extendsBitmap, index);
++ }
++ m->internalClass.set(engine, engine->internalClasses(EngineBase::Class_MemberData));
++ o->memberData.set(engine, m);
++ Q_ASSERT(o->memberData->internalClass);
++ m->values.alloc = static_cast<uint>((memberSize - sizeof(Heap::MemberData) + sizeof(Value))/sizeof(Value));
++ m->values.size = o->memberData->values.alloc;
++ m->init();
++ }
++
++ return o;
++ }
++
++ MarkStack::MarkStack(ExecutionEngine *engine)
++ : m_engine(engine)
++ {
++ m_base = (Heap::Base **)engine->gcStack->base();
++ m_top = m_base;
++ const size_t size = engine->maxGCStackSize() / sizeof(Heap::Base);
++ m_hardLimit = m_base + size;
++ m_softLimit = m_base + size * 3 / 4;
++ }
++
++ void MarkStack::drain()
++ {
++ // we're not calling drain(QDeadlineTimer::Forever) as that has higher overhead
++ while (m_top > m_base) {
++ Heap::Base *h = pop();
++ Q_ASSERT(h); // at this point we should only have Heap::Base objects in this area on the stack. If not, weird things might happen.
++ Q_ASSERT(h->internalClass);
++ h->internalClass->vtable->markObjects(h, this);
++ }
++ }
++
++ MarkStack::DrainState MarkStack::drain(QDeadlineTimer deadline)
++ {
++ do {
++ for (int i = 0; i <= MarkLoopIterationCountForDrain; ++i) {
++ if (m_top == m_base)
++ return DrainState::Complete;
++ Heap::Base *h = pop();
++ Q_ASSERT(h); // at this point we should only have Heap::Base objects in this area on the stack. If not, weird things might happen.
++ Q_ASSERT(h->internalClass);
++ h->internalClass->vtable->markObjects(h, this);
++ }
++ } while (!deadline.hasExpired());
++ return DrainState::Ongoing;
++ }
++
++ void MarkStack::setSoftLimit(size_t size)
++ {
++ m_softLimit = m_base + size;
++ Q_ASSERT(m_softLimit < m_hardLimit);
++ }
++
++ void MemoryManager::onEventLoop()
++ {
++ if (engine->inShutdown)
++ return;
++ if (gcBlocked == InCriticalSection) {
++ QMetaObject::invokeMethod(engine->publicEngine, [this]{
++ onEventLoop();
++ }, Qt::QueuedConnection);
++ return;
++ }
++ if (!gcStateMachine->inProgress())
++ return;
++
++ if (collectorStatistics) {
++ collectorStatistics->step(this);
++ if (!gcStateMachine->inProgress())
++ collectorStatistics->end(this);
++ } else {
++ gcStateMachine->step();
++ }
++ }
++
++
++ void MemoryManager::setGCTimeLimit(int timeMs)
++ {
++ gcStateMachine->timeLimit = std::chrono::milliseconds(timeMs);
++ }
++
++ void MemoryManager::sweep(bool lastSweep)
++ {
++
++ for (PersistentValueStorage::Iterator it = m_weakValues->begin(); it != m_weakValues->end(); ++it) {
++ Managed *m = (*it).managed();
++ if (!m || m->markBit())
++ continue;
++ // we need to call destroyObject on qobjectwrappers now, so that they can emit the destroyed
++ // signal before we start sweeping the heap
++ if (QObjectWrapper *qobjectWrapper = (*it).as<QObjectWrapper>()) {
++ qobjectWrapper->destroyObject(lastSweep);
++ }
++ }
++
++ freeWeakMaps(this);
++ freeWeakSets(this);
++
++ cleanupDeletedQObjectWrappersInSweep();
++
++ if (!lastSweep) {
++ engine->identifierTable->sweep();
++ blockAllocator.sweep(/*classCountPtr*/);
++ hugeItemAllocator.sweep();
++ icAllocator.sweep(/*classCountPtr*/);
++ }
++
++ // reset all black bits
++ blockAllocator.resetBlackBits();
++ hugeItemAllocator.resetBlackBits();
++ icAllocator.resetBlackBits();
++
++ usedSlotsAfterLastFullSweep = blockAllocator.usedSlotsAfterLastSweep + icAllocator.usedSlotsAfterLastSweep;
++ updateUnmanagedHeapSizeGCLimit();
++ gcBlocked = MemoryManager::Unblocked;
++ }
++
++ /*
++ \internal
++ Helper function used in sweep to clean up the (to-be-freed) QObjectWrapper
++ Used both in MemoryManager::sweep, and the corresponding gc statemachine phase
++ */
++ void MemoryManager::cleanupDeletedQObjectWrappersInSweep()
++ {
++ // onDestruction handlers may have accessed other QObject wrappers and reset their value, so ensure
++ // that they are all set to undefined.
++ for (PersistentValueStorage::Iterator it = m_weakValues->begin(); it != m_weakValues->end(); ++it) {
++ Managed *m = (*it).managed();
++ if (!m || m->markBit())
++ continue;
++ (*it) = Value::undefinedValue();
++ }
++
++ // Now it is time to free QV4::QObjectWrapper Value, we must check the Value's tag to make sure its object has been destroyed
++ const int pendingCount = m_pendingFreedObjectWrapperValue.size();
++ if (pendingCount) {
++ QList<Value *> remainingWeakQObjectWrappers;
++ remainingWeakQObjectWrappers.reserve(pendingCount);
++ for (int i = 0; i < pendingCount; ++i) {
++ Value *v = m_pendingFreedObjectWrapperValue.at(i);
++ if (v->isUndefined() || v->isEmpty())
++ PersistentValueStorage::free(v);
++ else
++ remainingWeakQObjectWrappers.append(v);
++ }
++ m_pendingFreedObjectWrapperValue = remainingWeakQObjectWrappers;
++ }
++
++ if (MultiplyWrappedQObjectMap *multiplyWrappedQObjects = engine->m_multiplyWrappedQObjects) {
++ for (MultiplyWrappedQObjectMap::Iterator it = multiplyWrappedQObjects->begin(); it != multiplyWrappedQObjects->end();) {
++ if (it.value().isNullOrUndefined())
++ it = multiplyWrappedQObjects->erase(it);
++ else
++ ++it;
++ }
++ }
++ }
++
++ bool MemoryManager::shouldRunGC() const
++ {
++ size_t total = blockAllocator.totalSlots() + icAllocator.totalSlots();
++ if (total > MinSlotsGCLimit && usedSlotsAfterLastFullSweep * GCOverallocation < total * 100)
++ return true;
++ return false;
++ }
++
++ static size_t dumpBins(BlockAllocator *b, const char *title)
++ {
++ const QLoggingCategory &stats = lcGcAllocatorStats();
++ size_t totalSlotMem = 0;
++ if (title)
++ qDebug(stats) << "Slot map for" << title << "allocator:";
++ for (uint i = 0; i < BlockAllocator::NumBins; ++i) {
++ uint nEntries = 0;
++ HeapItem *h = b->freeBins[i];
++ while (h) {
++ ++nEntries;
++ totalSlotMem += h->freeData.availableSlots;
++ h = h->freeData.next;
++ }
++ if (title)
++ qDebug(stats) << " number of entries in slot" << i << ":" << nEntries;
++ }
++ SDUMP() << " large slot map";
++ HeapItem *h = b->freeBins[BlockAllocator::NumBins - 1];
++ while (h) {
++ SDUMP() << " " << Qt::hex << (quintptr(h)/32) << h->freeData.availableSlots;
++ h = h->freeData.next;
++ }
++
++ if (title)
++ qDebug(stats) << " total mem in bins" << totalSlotMem*Chunk::SlotSize;
++ return totalSlotMem*Chunk::SlotSize;
++ }
++
++ /*!
++ \internal
++ Precondition: Incremental garbage collection must be currently active
++ Finishes incremental garbage collection, unless in a critical section
++ Code entering a critical section is expected to check if we need to
++ force a gc completion, and to trigger the gc again if necessary
++ when exiting the critcial section.
++ Returns \c true if the gc cycle completed, false otherwise.
++ */
++ bool MemoryManager::tryForceGCCompletion()
++ {
++ if (gcBlocked == InCriticalSection) {
++ qCDebug(lcGcForcedRuns)
++ << "Tried to force the GC to complete a run but failed due to being in a critical section.";
++ return false;
++ }
++
++ const bool incrementalGCIsAlreadyRunning = m_markStack != nullptr;
++ Q_ASSERT(incrementalGCIsAlreadyRunning);
++
++ qCDebug(lcGcForcedRuns) << "Forcing the GC to complete a run.";
++
++ auto oldTimeLimit = std::exchange(gcStateMachine->timeLimit, std::chrono::microseconds::max());
++ if (collectorStatistics) {
++ while (gcStateMachine->inProgress())
++ collectorStatistics->step(this);
++ collectorStatistics->end(this);
++ } else {
++ while (gcStateMachine->inProgress())
++ gcStateMachine->step();
++ }
++
++ gcStateMachine->timeLimit = oldTimeLimit;
++ return true;
++ }
++
++ void MemoryManager::runFullGC()
++ {
++ runGC();
++ if (m_markStack != nullptr)
++ tryForceGCCompletion();
++ }
++
++ void MemoryManager::runGC()
++ {
++ if (gcBlocked != Unblocked) {
++ return;
++ }
++
++ gcBlocked = MemoryManager::NormalBlocked;
++
++ if (statistics) {
++ statistics->maxAllocatedMem
++ = qMax(statistics->maxAllocatedMem, getAllocatedMem());
++ statistics->maxUsedBeforeGC
++ = qMax(statistics->maxUsedBeforeGC, getRegularItemsMem() + getLargeItemsMem());
++ }
++
++ if (collectorStatistics) {
++ if (!gcStateMachine->inProgress())
++ collectorStatistics->start(this);
++ collectorStatistics->step(this);
++ if (!gcStateMachine->inProgress())
++ collectorStatistics->end(this);
++ } else {
++ gcStateMachine->step();
++ }
++
++
++ if (statistics) {
++ statistics->maxUsedAfterGC
++ = qMax(statistics->maxUsedAfterGC, getRegularItemsMem() + getLargeItemsMem());
++ }
++ }
++
++ size_t MemoryManager::getRegularItemsMem() const
++ {
++ return blockAllocator.usedMem() + icAllocator.usedMem();
++ }
++
++ size_t MemoryManager::getAllocatedMem() const
++ {
++ return blockAllocator.allocatedMem() + icAllocator.allocatedMem() + hugeItemAllocator.usedMem();
++ }
++
++ size_t MemoryManager::getLargeItemsMem() const
++ {
++ return hugeItemAllocator.usedMem();
++ }
++
++ void MemoryManager::updateUnmanagedHeapSizeGCLimit()
++ {
++ if (3*unmanagedHeapSizeGCLimit <= 4 * unmanagedHeapSize) {
++ // more than 75% full, raise limit
++ unmanagedHeapSizeGCLimit = std::max(unmanagedHeapSizeGCLimit,
++ unmanagedHeapSize) * 2;
++ } else if (unmanagedHeapSize * 4 <= unmanagedHeapSizeGCLimit) {
++ // less than 25% full, lower limit
++ unmanagedHeapSizeGCLimit = qMax(std::size_t(MinUnmanagedHeapSizeGCLimit),
++ unmanagedHeapSizeGCLimit/2);
++ }
++
++ if (aggressiveGC && !engine->inShutdown) {
++ // ensure we don't 'loose' any memory
++ // but not during shutdown, because than we skip parts of sweep
++ // and use freeAll instead
++ Q_ASSERT(blockAllocator.allocatedMem()
++ == blockAllocator.usedMem() + dumpBins(&blockAllocator, nullptr));
++ Q_ASSERT(icAllocator.allocatedMem()
++ == icAllocator.usedMem() + dumpBins(&icAllocator, nullptr));
++ }
++ }
++
++ void MemoryManager::registerWeakMap(Heap::MapObject *map)
++ {
++ map->nextWeakMap = weakMaps;
++ weakMaps = map;
++ }
++
++ void MemoryManager::registerWeakSet(Heap::SetObject *set)
++ {
++ set->nextWeakSet = weakSets;
++ weakSets = set;
++ }
++
++ MemoryManager::~MemoryManager()
++ {
++ delete m_persistentValues;
++ dumpStats();
++
++ // do one last non-incremental sweep to clean up C++ objects
++ // first, abort any on-going incremental gc operation
++ setGCTimeLimit(-1);
++ if (engine->isGCOngoing) {
++ engine->isGCOngoing = false;
++ // Aborting the incremental run drops the mark stack, so we must also clear the
++ // blocked state. Otherwise the final sweep below runs destruction handlers with
++ // gcBlocked still set, and any GCCriticalSection they open would try to mark
++ // through the now-absent mark stack.
++ gcBlocked = MemoryManager::Unblocked;
++ m_markStack.reset();
++ gcStateMachine->state = GCState::Invalid;
++ blockAllocator.resetBlackBits();
++ hugeItemAllocator.resetBlackBits();
++ icAllocator.resetBlackBits();
++ }
++ // then sweep
++ sweep(/*lastSweep*/true);
++
++ blockAllocator.freeAll();
++ hugeItemAllocator.freeAll();
++ icAllocator.freeAll();
++
++ delete m_weakValues;
++ #ifdef V4_USE_VALGRIND
++ VALGRIND_DESTROY_MEMPOOL(this);
++ #endif
++ delete chunkAllocator;
++ }
++
++
++ void MemoryManager::dumpStats() const
++ {
++ if (!statistics)
++ return;
++
++ const QLoggingCategory &stats = lcGcStats();
++ qDebug(stats) << "Qml GC memory allocation statistics:";
++ qDebug(stats) << "Total memory allocated:" << statistics->maxAllocatedMem;
++ qDebug(stats) << "Max memory used before a GC run:" << statistics->maxUsedBeforeGC;
++ qDebug(stats) << "Max memory used after a GC run:" << statistics->maxUsedAfterGC;
++ qDebug(stats) << "Requests for different item sizes:";
++ for (int i = 1; i < BlockAllocator::NumBins - 1; ++i)
++ qDebug(stats) << " <" << (i << Chunk::SlotSizeShift) << " bytes: " << statistics->allocations[i];
++ qDebug(stats) << " >=" << ((BlockAllocator::NumBins - 1) << Chunk::SlotSizeShift) << " bytes: " << statistics->allocations[BlockAllocator::NumBins - 1];
++ }
++
++ void MemoryManager::collectFromJSStack(MarkStack *markStack) const
++ {
++ Value *v = engine->jsStackBase;
++ Value *top = engine->jsStackTop;
++ while (v < top) {
++ Managed *m = v->managed();
++ if (m) {
++ Q_ASSERT(m->inUse());
++ // Skip pointers to already freed objects, they are bogus as well
++ m->mark(markStack);
++ }
++ ++v;
++ }
++
++ for (auto *frame = engine->currentStackFrame; frame; frame = frame->parentFrame()) {
++ if (frame->isJSTypesFrame()) {
++ // Generator frames don't live on the JS stack, but in a heap object that the
++ // interpreter writes to without write barrier.
++ if (frame->v4Function && frame->v4Function->isGenerator())
++ static_cast<const JSTypesStackFrame *>(frame)->markJSFrame(markStack);
++ continue;
++ }
++
++ if (const QQmlPrivate::AOTTrackedLocalsStorage *locals
++ = static_cast<const MetaTypesStackFrame *>(frame)->locals()) {
++ // Actual AOT-compiled functions initialize the locals firsth thing when they
++ // are called. However, the ScopedStackFrame has no locals, but still uses a
++ // MetaTypesStackFrame.
++ locals->markObjects(markStack);
++ }
++ }
++ }
++
++ GCStateMachine::GCStateMachine()
++ : collectTimings(lcGcStepExecution().isDebugEnabled())
++ {
++ // base assumption: target 60fps, use at most 1/3 of time for gc
++ // unless overridden by env variable
++ bool ok = false;
++ auto envTimeLimit = qEnvironmentVariableIntValue("QV4_GC_TIMELIMIT", &ok );
++ if (!ok)
++ envTimeLimit = (1000 / 60) / 3;
++ if (envTimeLimit > 0)
++ timeLimit = std::chrono::milliseconds { envTimeLimit };
++ else
++ timeLimit = std::chrono::milliseconds { 0 };
++ }
++
++ static void logStepTiming(GCStateMachine* that, quint64 timing) {
++ auto registerTimingWithResetOnOverflow = [](
++ GCStateMachine::StepTiming& storage, quint64 timing, GCState state
++ ) {
++ auto wouldOverflow = [](quint64 lhs, quint64 rhs) {
++ return rhs > 0 && lhs > std::numeric_limits<quint64>::max() - rhs;
++ };
++
++ if (wouldOverflow(storage.rolling_sum, timing) || wouldOverflow(storage.count, 1)) {
++ qDebug(lcGcStepExecution) << "Resetting timings storage for"
++ << QMetaEnum::fromType<GCState>().key(state) << "due to overflow.";
++ storage.rolling_sum = timing;
++ storage.count = 1;
++ } else {
++ storage.rolling_sum += timing;
++ storage.count += 1;
++ }
++ };
++
++ GCStateMachine::StepTiming& storage = that->executionTiming[that->state];
++ registerTimingWithResetOnOverflow(storage, timing, that->state);
++
++ qDebug(lcGcStepExecution) << "Performed" << QMetaEnum::fromType<GCState>().key(that->state)
++ << "in" << timing << "microseconds";
++ qDebug(lcGcStepExecution) << "This step was performed" << storage.count << " time(s), executing in"
++ << (storage.rolling_sum / storage.count) << "microseconds on average.";
++ }
++
++ static GCState executeWithLoggingIfEnabled(GCStateMachine* that, GCStateInfo& stateInfo) {
++ if (!that->collectTimings)
++ return stateInfo.execute(that, that->stateData);
++
++ QElapsedTimer timer;
++ timer.start();
++ GCState next = stateInfo.execute(that, that->stateData);
++ logStepTiming(that, timer.nsecsElapsed()/1000);
++ return next;
++ }
++
++ static void redrainDuringSweep(GCStateMachine *that)
++ {
++ if (that->state > GCState::InitCallDestroyObjects) {
++ /* initCallDestroyObjects is the last action which drains the mark
++ stack by default. But as our write-barrier might end up putting
++ objects on the markStack which still reference other objects.
++ Especially when we call user code triggered by Component.onDestruction,
++ but also when we run into a timeout.
++ We don't redrain before InitCallDestroyObjects, as that would
++ potentially lead to useless busy-work (e.g., if the last referencs
++ to objects are removed while the mark phase is running)
++ */
++ redrain(that);
++ }
++ }
++
++ void GCStateMachine::transition() {
++ if (timeLimit.count() > 0) {
++ deadline = QDeadlineTimer(timeLimit);
++ bool deadlineExpired = false;
++ do {
++ redrainDuringSweep(this);
++ qCDebug(lcGcStateTransitions) << "Preparing to execute the"
++ << QMetaEnum::fromType<GCState>().key(state) << "state";
++ GCStateInfo& stateInfo = stateInfoMap[int(state)];
++ state = executeWithLoggingIfEnabled(this, stateInfo);
++ qCDebug(lcGcStateTransitions) << "Transitioning to the"
++ << QMetaEnum::fromType<GCState>().key(state) << "state";
++ if (stateInfo.breakAfter)
++ break;
++ } while (!(deadlineExpired = deadline.hasExpired()) && state != GCState::Invalid);
++ if (deadlineExpired)
++ handleTimeout(state);
++ if (state != GCState::Invalid)
++ QMetaObject::invokeMethod(mm->engine->publicEngine, [this]{
++ mm->onEventLoop();
++ }, Qt::QueuedConnection);
++ } else {
++ deadline = QDeadlineTimer::Forever;
++ while (state != GCState::Invalid) {
++ redrainDuringSweep(this);
++ qCDebug(lcGcStateTransitions) << "Preparing to execute the"
++ << QMetaEnum::fromType<GCState>().key(state) << "state";
++ GCStateInfo& stateInfo = stateInfoMap[int(state)];
++ state = executeWithLoggingIfEnabled(this, stateInfo);
++ qCDebug(lcGcStateTransitions) << "Transitioning to the"
++ << QMetaEnum::fromType<GCState>().key(state) << "state";
++ }
++ }
++ }
++
++ std::vector<QObject *> MemoryManager::findObjectsForCompilationUnits(
++ std::vector<QQmlRefPointer<QV4::CompiledData::CompilationUnit>> &&units)
++ {
++
++ ObjectsForCompilationUnit recorded { std::move(units), {} };
++ m_recordedObjects = &recorded;
++
++ runFullGC();
++
++ m_recordedObjects = nullptr;
++ return std::exchange(recorded.objects, {});
++ }
++
++ void MemoryManager::CollectorStatistics::start(MemoryManager *mm)
++ {
++ const QLoggingCategory &stats = lcGcAllocatorStats();
++
++ oldUnmanagedSize = mm->unmanagedHeapSize;
++ regularItemsBefore = mm->getRegularItemsMem();
++ largeItemsBefore = mm->getLargeItemsMem();
++ oldChunks = mm->blockAllocator.chunks.size();
++ triggeredByUnmanagedHeap = (mm->unmanagedHeapSize > mm->unmanagedHeapSizeGCLimit);
++
++ qDebug(stats) << "========== GC ==========";
++ #ifdef MM_STATS
++ qDebug(stats) << " Triggered by alloc request of" << mm->lastAllocRequestedSlots << "slots.";
++ qDebug(stats) << " Allocations since last GC" << mm->allocationCount;
++ mm->allocationCount = 0;
++ #endif
++ const size_t allocatedMem = mm->getAllocatedMem();
++ qDebug(stats) << "Allocated" << allocatedMem << "bytes in" << oldChunks << "chunks";
++ qDebug(stats) << "Fragmented memory before GC" << (allocatedMem - regularItemsBefore);
++ dumpBins(&mm->blockAllocator, "Block");
++ dumpBins(&mm->icAllocator, "InternalClass");
++ }
++
++ void MemoryManager::CollectorStatistics::step(MemoryManager *mm)
++ {
++ QElapsedTimer t;
++ t.start();
++ mm->gcStateMachine->step();
++ gcTime += t.nsecsElapsed();
++ }
++
++ void MemoryManager::CollectorStatistics::end(MemoryManager *mm)
++ {
++ const QLoggingCategory &stats = lcGcAllocatorStats();
++
++ const size_t regularItemsAfter = mm->getRegularItemsMem();
++ const size_t largeItemsAfter = mm->getLargeItemsMem();
++
++ if (triggeredByUnmanagedHeap) {
++ qDebug(stats) << "triggered by unmanaged heap:";
++ qDebug(stats) << " old unmanaged heap size:" << oldUnmanagedSize;
++ qDebug(stats) << " new unmanaged heap:" << mm->unmanagedHeapSize;
++ qDebug(stats) << " unmanaged heap limit:" << mm->unmanagedHeapSizeGCLimit;
++ }
++ const size_t memInBins = dumpBins(&mm->blockAllocator, "Block")
++ + dumpBins(&mm->icAllocator, "InternalClasss");
++ qDebug(stats) << "Garbage collection took" << (gcTime / 1000) << "us.";
++
++ qDebug(stats) << "Regular item memory before GC:" << regularItemsBefore;
++ qDebug(stats) << "Regular item memory after GC:" << regularItemsAfter;
++ qDebug(stats) << "Freed up bytes :" << (regularItemsBefore - regularItemsAfter);
++ qDebug(stats) << "Freed up chunks :" << (oldChunks - mm->blockAllocator.chunks.size());
++ const size_t lost = mm->blockAllocator.allocatedMem() + mm->icAllocator.allocatedMem()
++ - memInBins - regularItemsAfter;
++ if (lost)
++ qDebug(stats) << "!!!!!!!!!!!!!!!!!!!!! LOST MEM:" << lost << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!";
++ if (largeItemsBefore || largeItemsAfter) {
++ qDebug(stats) << "Large item memory before GC:" << largeItemsBefore;
++ qDebug(stats) << "Large item memory after GC:" << largeItemsAfter;
++ qDebug(stats) << "Large item memory freed up:" << (largeItemsBefore - largeItemsAfter);
++ }
++
++ qDebug(stats) << "======== End GC ========";
++ }
++
++ } // namespace QV4
++
++ QT_END_NAMESPACE
++
++ #include "moc_qv4mm_p.cpp"