Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qsamplecache_p.h"
#include <QtMultimedia/qaudiobuffer.h>
#include <QtMultimedia/qaudiodecoder.h>
#include <QtMultimedia/private/qaudiohelpers_p.h>
#include <QtMultimedia/private/qdrawavaudiodecoder_p.h>
#include <QtMultimedia/private/qmultimediautils_p.h>
#include <QtConcurrent/qtconcurrentrun.h>
#include <QtCore/qapplicationstatic.h>
#include <QtCore/qbuffer.h>
#include <QtCore/qcoreapplication.h>
#include <QtCore/qdebug.h>
#include <QtCore/qeventloop.h>
#include <QtCore/qfile.h>
#include <QtCore/qfuturewatcher.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qmutex.h>
#if QT_CONFIG(network)
# include <QtNetwork/qnetworkaccessmanager.h>
# include <QtNetwork/qnetworkreply.h>
# include <QtNetwork/qnetworkrequest.h>
#endif
#include "dr_wav.h"
#include <utility>
Q_STATIC_LOGGING_CATEGORY(qLcSampleCache, "qt.multimedia.samplecache")
#if !QT_CONFIG(thread)
# define thread_local
#endif
QT_BEGIN_NAMESPACE
QSample::QSample(QUrl url, QSampleCache *parent) : m_parent(parent), m_url(std::move(url)) { }
QSample::QSample(QUrl url, QSampleCache *parent, std::optional<SampleRate> targetSampleRate)
: m_parent(parent), m_url(std::move(url)), m_targetSampleRate(targetSampleRate)
{
}
QSample::~QSample()
{
// Remove ourselves from our parent
if (m_parent)
m_parent->removeUnreferencedSample(m_url, m_targetSampleRate);
qCDebug(qLcSampleCache) << "~QSample" << this << ": deleted [" << m_url << "]" << QThread::currentThread();
}
void QSample::setError()
{
m_state = State::Error;
}
void QSample::setData(QByteArray data, QAudioFormat format)
{
m_state = State::Ready;
m_soundData = std::move(data);
m_audioFormat = format;
}
QSample::State QSample::state() const
{
return m_state;
}
void QSample::clearParent()
{
m_parent = nullptr;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
Q_APPLICATION_STATIC(QSampleCache, sampleCache)
QSampleCache *QSampleCache::instance()
{
return sampleCache();
}
#if QT_CONFIG(thread)
QThreadPool *QSampleCache::threadPool()
{
#ifdef Q_OS_WASM
return QThreadPool::globalInstance();
#else
return &m_threadPool;
#endif
}
#endif
QSampleCache::QSampleCache(QObject *parent) : QObject(parent)
{
#if QT_CONFIG(thread)
if (!thread()->isMainThread())
moveToThread(qApp->thread());
# if !defined(Q_OS_WASM)
// we limit the number of loader threads to avoid thread explosion
static constexpr int loaderThreadLimit = 8;
m_threadPool.setObjectName("QSampleCachePool");
m_threadPool.setMaxThreadCount(loaderThreadLimit);
m_threadPool.setExpiryTimeout(15);
m_threadPool.setThreadPriority(QThread::LowPriority);
m_threadPool.setServiceLevel(QThread::QualityOfService::Eco);
qAddPostRoutine([] {
// HACK: we need to stop the thread pool before qApp is nulled, otherwise some threads might still try construct
// some Q_APPLICATION_STATIC instances, causing assertion failures inside QNetworkAccessManager
Q_ASSERT(qApp && "QApplication is still valid");
QSampleCache *instance = sampleCache();
instance->m_threadPool.clear();
instance->m_threadPool.waitForDone();
});
# endif // Q_OS_WASM
#endif // QT_CONFIG(thread)
}
QSampleCache::~QSampleCache()
{
#if QT_CONFIG(thread) && !defined(Q_OS_WASM)
m_threadPool.clear();
m_threadPool.waitForDone();
#endif
for (auto &entry : m_loadedSamples) {
auto samplePtr = entry.second.lock();
if (samplePtr)
samplePtr->clearParent();
}
for (auto &entry : m_pendingSamples) {
auto samplePtr = entry.second.first;
if (samplePtr)
samplePtr->clearParent();
}
}
QSampleCache::SampleLoadResult QSampleCache::loadSample(QSpan<const char> data)
{
using namespace QtPrivate;
drwav wavParser;
bool success = drwav_init_memory(&wavParser, data.data(), data.size(), nullptr);
if (!success)
return q23::unexpected(QSampleLoadError::FormatError);
auto cleanup = qScopeGuard([&] {
drwav_uninit(&wavParser);
});
// using float as internal format. one could argue to use int16 and save half the ram at the
// cost of potential run-time conversions
QAudioFormat audioFormat;
audioFormat.setChannelCount(wavParser.channels);
audioFormat.setSampleFormat(QAudioFormat::Float);
audioFormat.setSampleRate(wavParser.sampleRate);
audioFormat.setChannelConfig(
QAudioFormat::defaultChannelConfigForChannelCount(wavParser.channels));
std::optional<QByteArray> sampleData = QtMultimediaPrivate::readFramesInChunks(
wavParser.totalPCMFrameCount, sizeof(float) * wavParser.channels,
[&](uint64_t frameCount, char *out) -> uint64_t {
return drwav_read_pcm_frames_f32(&wavParser, frameCount, reinterpret_cast<float *>(out));
});
if (!sampleData)
return q23::unexpected(QSampleLoadError::FormatError);
return std::pair{
std::move(*sampleData),
audioFormat,
};
}
namespace {
QByteArray convertToFloat32(const QByteArray &data, const QAudioFormat &fmt)
{
if (fmt.sampleFormat() == QAudioFormat::Float)
return data;
int totalSamples = fmt.framesForBytes(data.size()) * fmt.channelCount();
QByteArray result{
totalSamples * int(sizeof(float)),
Qt::Initialization::Uninitialized,
};
using namespace QAudioHelperInternal;
convertSampleFormat(as_bytes(QSpan{ data }), toNativeSampleFormat(fmt.sampleFormat()),
as_writable_bytes(QSpan{ result }), NativeSampleFormat::float32_t);
return result;
}
QSampleCache::SampleLoadResult runDecoderLoop(QAudioDecoder &decoder, QEventLoop &loop)
{
using SampleLoadResult = QSampleCache::SampleLoadResult;
QByteArray accumulated;
QAudioFormat fmt;
SampleLoadResult result = q23::unexpected(QSampleLoadError::DecoderError);
QObject::connect(&decoder, &QAudioDecoder::bufferReady, &loop, [&] {
QAudioBuffer buf = decoder.read();
if (!buf.isValid())
return;
if (!fmt.isValid())
fmt = buf.format();
accumulated.append(buf.constData<char>(), buf.byteCount());
});
QObject::connect(&decoder, &QAudioDecoder::finished, &loop, [&] {
QByteArray floatData = convertToFloat32(accumulated, fmt);
QAudioFormat floatFmt = fmt;
floatFmt.setSampleFormat(QAudioFormat::Float);
result = std::pair{ std::move(floatData), floatFmt };
loop.quit();
});
QObject::connect(&decoder, qOverload<QAudioDecoder::Error>(&QAudioDecoder::error),
&loop, [&](QAudioDecoder::Error) {
result = q23::unexpected(QSampleLoadError::DecoderError);
loop.quit();
});
decoder.start();
if (decoder.error() != QAudioDecoder::NoError)
return q23::unexpected(QSampleLoadError::DecoderError);
loop.exec(QEventLoop::ExcludeUserInputEvents);
return result;
}
} // unnamed namespace
QSampleCache::SampleLoadResult
QSampleCache::loadSampleViaDecoder(std::variant<QUrl, QByteArray> arg)
{
// caveat: we run our own event loop, so this function should ideally not be run from the main thread.
using namespace QtMultimediaPrivate;
QAudioDecoder decoder;
if (!decoder.isSupported())
return q23::unexpected(QSampleLoadError::NotSupported);
QEventLoop loop;
// clang-format off
return std::visit(qOverloadedVisitor([&](QUrl url) -> SampleLoadResult {
decoder.setSource(url);
return runDecoderLoop(decoder, loop);
}, [&](QByteArray data) -> SampleLoadResult {
QBuffer buffer;
buffer.setData(data);
if (!buffer.open(QIODevice::ReadOnly))
return q23::unexpected(QSampleLoadError::IoError);
decoder.setSourceDevice(&buffer);
return runDecoderLoop(decoder, loop);
}), std::move(arg));
// clang-format on
}
#if QT_CONFIG(network)
namespace {
Q_CONSTINIT thread_local std::optional<QNetworkAccessManager> g_networkAccessManager;
QNetworkAccessManager &threadLocalNetworkAccessManager()
{
if (!g_networkAccessManager.has_value()) {
g_networkAccessManager.emplace();
if (QThread::isMainThread()) {
// poor man's Q_APPLICATION_STATIC
qAddPostRoutine([] {
g_networkAccessManager.reset();
});
}
}
return *g_networkAccessManager;
}
} // namespace
#endif
#if QT_CONFIG(thread)
QSampleCache::SampleLoadResult
QSampleCache::loadSample(const QUrl &url, std::optional<SampleSourceType> forceSourceType)
{
using namespace Qt::Literals;
SampleSourceType realSourceType =
forceSourceType.value_or(url.scheme() == u"qrc"_s || url.scheme() == u"file"_s
? SampleSourceType::File
: SampleSourceType::NetworkManager);
if (realSourceType == SampleSourceType::AudioDecoder)
return loadSampleViaDecoder(url);
if (url.scheme().isEmpty())
// exit early, to avoid QNetworkAccessManager trying to construct a default ssl
// configuration, which tends to cause timeouts on CI on macos.
// catch this case and exit early.
return q23::unexpected(QSampleLoadError::IoError);
bool errorOccurred = false;
std::unique_ptr<QIODevice> decoderInput;
if (realSourceType == SampleSourceType::File) {
QString locationString =
url.isLocalFile() ? url.toLocalFile() : u":" + url.toString(QUrl::RemoveScheme);
auto *file = new QFile(locationString);
bool opened = file->open(QFile::ReadOnly);
if (!opened)
errorOccurred = true;
decoderInput.reset(file);
} else {
#if QT_CONFIG(network)
QNetworkReply *reply = threadLocalNetworkAccessManager().get(QNetworkRequest(url));
if (reply->error() != QNetworkReply::NoError)
errorOccurred = true;
connect(reply, &QNetworkReply::errorOccurred, reply,
[&]([[maybe_unused]] QNetworkReply::NetworkError errorCode) {
errorOccurred = true;
});
decoderInput.reset(reply);
#else
return q23::unexpected(QSampleLoadError::IoError);
#endif
}
if (!decoderInput->isOpen())
return q23::unexpected(QSampleLoadError::IoError);
QByteArray data = decoderInput->readAll();
if (data.isEmpty() || errorOccurred)
return q23::unexpected(QSampleLoadError::IoError);
SampleLoadResult result = loadSample(data);
if (!result && result.error() == QSampleLoadError::FormatError && !forceSourceType) {
qCDebug(qLcSampleCache) << "drwav failed, retrying via QAudioDecoder";
result = loadSampleViaDecoder(data);
if (!result) {
qCDebug(qLcSampleCache) << "device-based decode failed, retrying via URL";
result = loadSampleViaDecoder(url);
}
}
return result;
}
#endif
QFuture<QSampleCache::SampleLoadResult>
QSampleCache::loadSampleAsync(const QUrl &url, std::optional<SampleSourceType> forceSourceType)
{
auto promise = std::make_shared<QPromise<QSampleCache::SampleLoadResult>>();
auto future = promise->future();
auto fulfilPromise = [&](auto &&result) mutable {
promise->start();
promise->addResult(result);
promise->finish();
};
using namespace Qt::Literals;
SampleSourceType realSourceType = (url.scheme() == u"qrc"_s || url.scheme() == u"file"_s)
? SampleSourceType::File
: SampleSourceType::NetworkManager;
if (realSourceType == SampleSourceType::File) {
QString locationString = url.toString(QUrl::RemoveScheme);
if (url.scheme() == u"qrc"_s)
locationString = u":" + locationString;
QFile file{ locationString };
bool opened = file.open(QFile::ReadOnly);
if (!opened) {
fulfilPromise(q23::unexpected(QSampleLoadError::IoError));
return future;
}
QByteArray data = file.readAll();
if (data.isEmpty()) {
fulfilPromise(q23::unexpected(QSampleLoadError::IoError));
return future;
}
if (forceSourceType == SampleSourceType::AudioDecoder)
return loadSampleAsyncViaDecoder(data);
auto drwavResult = loadSample(data);
if (drwavResult || drwavResult.error() != QSampleLoadError::FormatError) {
fulfilPromise(drwavResult);
return future;
}
return loadSampleAsyncViaDecoder(data);
}
#if QT_CONFIG(network)
QNetworkReply *reply = threadLocalNetworkAccessManager().get(QNetworkRequest(url));
if (reply->error() != QNetworkReply::NoError) {
fulfilPromise(q23::unexpected(QSampleLoadError::IoError));
reply->deleteLater();
return future;
}
connect(reply, &QNetworkReply::errorOccurred, reply,
[reply, promise]([[maybe_unused]] QNetworkReply::NetworkError errorCode) {
promise->start();
promise->addResult(q23::unexpected(QSampleLoadError::IoError));
promise->finish();
reply->deleteLater(); // we cannot delete immediately
});
connect(reply, &QNetworkReply::finished, reply,
[promise, reply, fulfilPromise = std::move(fulfilPromise), forceSourceType]() mutable {
QByteArray data = reply->readAll();
if (data.isEmpty()) {
promise->start();
promise->addResult(q23::unexpected(QSampleLoadError::IoError));
promise->finish();
} else if (forceSourceType == SampleSourceType::AudioDecoder) {
auto decoderFuture = loadSampleAsyncViaDecoder(data);
decoderFuture.then([promise](SampleLoadResult result) {
promise->start();
promise->addResult(std::move(result));
promise->finish();
});
} else {
auto drwavResult = loadSample(data);
if (drwavResult || drwavResult.error() != QSampleLoadError::FormatError) {
fulfilPromise(drwavResult);
} else {
auto decoderFuture = loadSampleAsyncViaDecoder(data);
decoderFuture.then([promise](SampleLoadResult result) {
promise->start();
promise->addResult(std::move(result));
promise->finish();
});
}
}
reply->deleteLater(); // we cannot delete immediately
});
#else
fulfilPromise(q23::unexpected(QSampleLoadError::IoError));
#endif
return future;
}
namespace {
// keeps the only reference to the QAudioDecoder instances alive until they finish decoding.
// We cannot keep the QAudioDecoder instances in the the decoder connections, as they may not
// fire if the application is shutting down before the decoder terminates
struct QStaticDecoderSingleton
{
void addDecoder(std::shared_ptr<QAudioDecoder> decoder)
{
std::lock_guard guard(mutex);
decoders.insert(std::move(decoder));
}
void removeDecoder(std::shared_ptr<QAudioDecoder> decoder)
{
std::lock_guard guard(mutex);
decoders.erase(decoder);
}
std::set<std::shared_ptr<QAudioDecoder>> decoders;
QBasicMutex mutex;
};
Q_APPLICATION_STATIC(QStaticDecoderSingleton, decoders);
} // namespace
QFuture<QSampleCache::SampleLoadResult>
QSampleCache::loadSampleAsyncViaDecoder(QByteArray data)
{
// NB: we heap-allocate the QAudioDecoder and keep it alive until decoding finishes or an error
// occurs. however we cannot keep the QAudioDecoder alive in the lambda captures of the decoder
// signals, as they might not fire if the application is shutting down before the decoder
// terminates, causing a potential leak. Instead, we keep the decoders in a singleton set until
// they finish decoding, and capture weak pointers to them in the lambda captures of the
// finished/error signals.
auto promise = std::make_shared<QPromise<SampleLoadResult>>();
auto future = promise->future();
auto decoder = QtMultimediaPrivate::makeSharedDeleteLater<QAudioDecoder>();
if (!decoder->isSupported()) {
promise->start();
promise->addResult(q23::unexpected(QSampleLoadError::NotSupported));
promise->finish();
return future;
}
auto *buffer = new QBuffer(decoder.get());
buffer->setData(data);
buffer->open(QIODevice::ReadOnly);
auto accum = std::make_shared<QByteArray>();
auto fmt = std::make_shared<QAudioFormat>();
decoders->addDecoder(decoder);
QObject::connect(decoder.get(), &QAudioDecoder::bufferReady, decoder.get(),
[weakDecoder = std::weak_ptr<QAudioDecoder>(decoder), accum, fmt] {
auto decoder = weakDecoder.lock();
if (!decoder)
return;
QAudioBuffer buf = decoder->read();
if (!buf.isValid())
return;
if (!fmt->isValid())
*fmt = buf.format();
accum->append(buf.constData<char>(), buf.byteCount());
});
QObject::connect(decoder.get(), &QAudioDecoder::finished, decoder.get(),
[promise, accum, fmt, weakDecoder = std::weak_ptr<QAudioDecoder>(decoder)] {
auto decoder = weakDecoder.lock();
if (!decoder)
return;
QByteArray floatData = convertToFloat32(*accum, *fmt);
QAudioFormat floatFmt = *fmt;
floatFmt.setSampleFormat(QAudioFormat::Float);
promise->start();
promise->addResult(std::pair{ std::move(floatData), floatFmt });
promise->finish();
decoders->removeDecoder(decoder);
});
QObject::connect(
decoder.get(), qOverload<QAudioDecoder::Error>(&QAudioDecoder::error), decoder.get(),
[promise, weakDecoder = std::weak_ptr<QAudioDecoder>(decoder)](QAudioDecoder::Error) {
auto decoder = weakDecoder.lock();
if (!decoder)
return;
promise->start();
promise->addResult(q23::unexpected(QSampleLoadError::DecoderError));
promise->finish();
decoders->removeDecoder(decoder);
});
decoder->setSourceDevice(buffer);
decoder->start();
return future;
}
bool QSampleCache::isCached(const QUrl &url, std::optional<int> targetSampleRate) const
{
std::lock_guard guard(m_mutex);
using namespace QtMultimediaPrivate;
const SampleKey key{ url, asSampleRate(targetSampleRate) };
return m_loadedSamples.find(key) != m_loadedSamples.end()
|| m_pendingSamples.find(key) != m_pendingSamples.end();
}
QFuture<SharedSamplePtr>
QSampleCache::requestSampleFuture(const QUrl &url, std::optional<int> targetSampleRate,
std::optional<SampleSourceType> forceSourceType)
{
std::lock_guard guard(m_mutex);
using namespace QtMultimediaPrivate;
auto targetRate = asSampleRate(targetSampleRate);
const SampleKey key{ url, targetRate };
auto promise = std::make_shared<QPromise<SharedSamplePtr>>();
auto future = promise->future();
// found and ready
auto found = m_loadedSamples.find(key);
if (found != m_loadedSamples.end()) {
SharedSamplePtr foundSample = found->second.lock();
Q_ASSERT(foundSample);
Q_ASSERT(foundSample->state() == QSample::Ready);
promise->start();
promise->addResult(std::move(foundSample));
promise->finish();
return future;
}
// already in the process of being loaded
auto pending = m_pendingSamples.find(key);
if (pending != m_pendingSamples.end()) {
pending->second.second.append(promise);
return future;
}
// we need to start a new load process
SharedSamplePtr sample = std::make_shared<QSample>(url, this, targetRate);
m_pendingSamples.emplace(key, std::pair{ sample, QList<SharedSamplePromise>{ promise } });
QFuture<SampleLoadResult> futureResult = [&] {
#if QT_CONFIG(thread)
if (threadPool()->maxThreadCount() > 0)
return QtConcurrent::run(threadPool(), [url, type = forceSourceType] {
return loadSample(url, type);
});
#endif
return loadSampleAsync(url, forceSourceType);
}();
futureResult.then(this,
[this, key, targetRate,
sample = std::move(sample)](SampleLoadResult loadResult) mutable {
if (loadResult) {
QByteArray sampleData = loadResult->first;
QAudioFormat sampleFormat = loadResult->second;
if (targetRate && sampleFormat.sampleRate() != qToUnderlying(*targetRate)) {
const int rate = qToUnderlying(*targetRate);
const qsizetype totalFloats = sampleData.size() / qsizetype(sizeof(float));
QSpan<const float> inputSpan{
reinterpret_cast<const float *>(sampleData.constData()),
totalFloats
};
sampleData = QAudioHelperInternal::resampleAudioCatmullRom(
inputSpan, sampleFormat.channelCount(),
sampleFormat.sampleRate(), rate);
sampleFormat.setSampleRate(rate);
}
sample->setData(std::move(sampleData), sampleFormat);
} else {
sample->setError();
}
std::lock_guard guard(m_mutex);
auto pending = m_pendingSamples.find(key);
if (pending != m_pendingSamples.end()) {
for (auto &promise : pending->second.second) {
promise->start();
promise->addResult(loadResult ? sample : nullptr);
promise->finish();
}
}
if (loadResult)
m_loadedSamples.emplace(key, sample);
if (pending != m_pendingSamples.end())
m_pendingSamples.erase(pending);
sample = {};
});
return future;
}
void QSampleCache::removeUnreferencedSample(const QUrl &url,
std::optional<SampleRate> targetSampleRate)
{
std::lock_guard guard(m_mutex);
m_loadedSamples.erase(SampleKey{ url, targetSampleRate });
}
QT_END_NAMESPACE
#if !QT_CONFIG(thread)
# undef thread_local
#endif