qaudioengine_withplayer.cpp

++ // Copyright (C) 2025 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only

++

++ #include "qaudioengine_withplayer_p.h"

++

++ #include <QtSpatialAudio/private/qambisonicdecoder_p.h>

++ #include <QtSpatialAudio/private/qspatialaudiosound_p.h>

++ #include <QtMultimedia/qmediadevices.h>

++ #include <QtMultimedia/private/qaudiosystem_p.h>

++ #include <QtMultimedia/private/qmemory_resource_tlsf_p.h>

++ #include <QtMultimedia/private/qmultimedia_ranges_p.h>

++ #include <QtMultimedia/private/qrtaudioengine_p.h>

++ #include <QtMultimedia/private/qtmm_pmr_p.h>

++

++ #include <resonance_audio.h>

++

++ QT_BEGIN_NAMESPACE

++

++ enum class SourceId : int { };

++

++ class QResonanceAudioPlayer : public QtMultimediaPrivate::QRtAudioEngineVoice

++ {

++ public:

++     using SharedPlaybackState = QAudioEnginePrivate::SharedPlaybackState;

++

++     QResonanceAudioPlayer(QAudioEngineWithPlayer *player, VoiceId id);

++

++     /// QRtAudioEngineVoice overrides

++     VoicePlayResult play(QSpan<float>) noexcept Q_DECL_NONBLOCKING_FUNCTION override;

++     bool isActive() noexcept Q_DECL_NONBLOCKING_FUNCTION override { return true; }

++     const QAudioFormat &format() noexcept override { return m_format; }

++

++     void addSound(SourceId, int numberOfChannels, SharedPlaybackState = nullptr);

++     void setSoundPlaybackData(SourceId, SharedPlaybackState);

++     void removeSound(SourceId);

++

++     void setPaused(bool);

++     void setOutputMode(QAudioEngine::OutputMode mode);

++

++ private:

++     void processSlice(QSpan<float> hostBuffer);

++     void processSliceFillResonanceBuffers();

++     void processSliceProcessSurroundMode(QSpan<float>);

++     void processSliceProcessWithoutReverb(QSpan<float>);

++

++     constexpr static auto framesPerBuffer = QAudioEnginePrivate::framesPerBuffer;

++     QAudioEngine::OutputMode m_outputMode;

++     const std::shared_ptr<vraudio::ResonanceAudio> m_resonanceAudio;

++     const QAudioFormat m_format;

++     QAmbisonicDecoder m_ambisonicDecoder;

++     bool m_paused = false;

++

++     struct SoundState

++     {

++         SharedPlaybackState playbackState;

++         int numberOfChannels;

++     };

++

++     static constexpr size_t poolSize =

++             sizeof(QtMultimediaPrivate::pmr::map<SourceId, SoundState>::node_type)

++                     * 512 // space for 512 sounds

++             + sizeof(float) * qToUnderlying(framesPerBuffer) * 32 // leftover buffer for 32 channels

++             + sizeof(float) * qToUnderlying(framesPerBuffer) * 32 // temporary slice buffer

++             + 16384; // some extra space for internal fragmentation and allocator metadata

++     QtMultimediaPrivate::QTlsfMemoryResource m_rtMemoryPool{ poolSize };

++

++     QtMultimediaPrivate::pmr::map<SourceId, SoundState> m_sounds{ &m_rtMemoryPool };

++     QtMultimediaPrivate::pmr::vector<float> m_leftoverBuffer{ &m_rtMemoryPool };

++ };

++

++ QResonanceAudioPlayer::QResonanceAudioPlayer(QAudioEngineWithPlayer *player, VoiceId id)

++     : QRtAudioEngineVoice(id),

++       m_outputMode(player->outputMode()),

++       m_resonanceAudio(player->resonanceAudio),

++       m_format(player->audioFormat()),

++       m_ambisonicDecoder{

++           QAmbisonicDecoder::HighQuality,

++           m_format.sampleRate(),

++           m_format.channelCount(),

++           m_format.channelConfig(),

++       }

++ {

++ }

++

++ QtMultimediaPrivate::QRtAudioEngineVoice::VoicePlayResult

++ QResonanceAudioPlayer::play(QSpan<float> outputBuffer) noexcept Q_DECL_NONBLOCKING_FUNCTION

++ {

++     using namespace QtMultimediaPrivate;

++

++     if (m_paused)

++         return VoicePlayResult::Playing;

++

++     const size_t samplesPerSlice = m_format.channelCount() * qToUnderlying(framesPerBuffer);

++

++     // resonance-audio renders slices with fill (not additive) semantics, so every slice is

++     // rendered into a zero-initialized scratch buffer first and mixed into outputBuffer

++     // afterwards

++     while (!outputBuffer.empty()) {

++         if (m_leftoverBuffer.empty() && size_t(outputBuffer.size()) >= samplesPerSlice) {

++             // fast path: the whole slice is consumed immediately, so we can use a stack-allocated

++             // scratch buffer instead of carrying it via m_leftoverBuffer

++             withTemporaryBuffer(samplesPerSlice * sizeof(float), [&](QSpan<std::byte> raw) {

++                 QSpan<float> scratch{

++                     reinterpret_cast<float *>(raw.data()),

++                     qsizetype(samplesPerSlice),

++                 };

++                 std::fill(scratch.begin(), scratch.end(), 0.0f);

++                 processSlice(scratch);

++                 for (size_t i = 0; i != samplesPerSlice; ++i)

++                     outputBuffer[i] += scratch[i];

++             });

++             outputBuffer = drop(outputBuffer, samplesPerSlice);

++             continue;

++         }

++

++         if (m_leftoverBuffer.empty()) {

++             m_leftoverBuffer.assign(samplesPerSlice, 0.0f);

++             processSlice(QSpan<float>(m_leftoverBuffer));

++         }

++

++         const auto samplesToMix = std::min<qsizetype>(m_leftoverBuffer.size(), outputBuffer.size());

++         for (qsizetype i = 0; i != samplesToMix; ++i)

++             outputBuffer[i] += m_leftoverBuffer[i];

++

++         m_leftoverBuffer.erase(m_leftoverBuffer.begin(), m_leftoverBuffer.begin() + samplesToMix);

++         outputBuffer = drop(outputBuffer, samplesToMix);

++     }

++

++     return VoicePlayResult::Playing;

++ }

++

++ void QResonanceAudioPlayer::addSound(SourceId id, int numberOfChannels, SharedPlaybackState state)

++ {

++     auto [_, inserted] = m_sounds.insert_or_assign(id,

++                                                    SoundState{

++                                                            std::move(state),

++                                                            numberOfChannels,

++                                                    });

++     Q_ASSERT(inserted);

++ }

++

++ void QResonanceAudioPlayer::setSoundPlaybackData(SourceId id, SharedPlaybackState state)

++ {

++     auto it = m_sounds.find(id);

++     Q_ASSERT(it != m_sounds.end());

++     it->second.playbackState = std::move(state);

++ }

++

++ void QResonanceAudioPlayer::removeSound(SourceId sound)

++ {

++     m_sounds.erase(sound);

++ }

++

++ void QResonanceAudioPlayer::setPaused(bool paused)

++ {

++     m_paused = paused;

++ }

++

++ void QResonanceAudioPlayer::setOutputMode(QAudioEngine::OutputMode mode)

++ {

++     m_outputMode = mode;

++ }

++

++ void QResonanceAudioPlayer::processSlice(QSpan<float> hostBuffer)

++ {

++     using QtMultimediaPrivate::drop;

++     constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);

++     const qsizetype samplesPerSlice = m_format.channelCount() * framesPerBuffer;

++

++     Q_ASSERT(hostBuffer.size() == samplesPerSlice);

++

++     // fill host API

++     processSliceFillResonanceBuffers();

++

++     // and process

++     switch (m_outputMode) {

++     case QAudioEngine::Surround:

++         processSliceProcessSurroundMode(hostBuffer);

++         break;

++     default:

++         processSliceProcessWithoutReverb(hostBuffer);

++         break;

++     }

++ }

++

++

++ void QResonanceAudioPlayer::processSliceFillResonanceBuffers()

++ {

++     constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);

++     static constexpr std::array<float, 2 * framesPerBuffer> nullBuffer{};

++     using QtMultimediaPrivate::take;

++     vraudio::ResonanceAudioApi *api = m_resonanceAudio->api.get();

++

++     for (auto &&[sourceId, sourceState] : m_sounds) {

++         int numberOfChannels = sourceState.numberOfChannels;

++         SharedPlaybackState &playbackState = sourceState.playbackState;

++         if (playbackState) {

++             Q_ASSERT(playbackState->format().channelCount() <= 2);

++             std::array<float, 2 * framesPerBuffer> buf;

++

++             playbackState->getBuffer(take(

++                     QSpan<float>{ buf }, playbackState->format().channelCount() * framesPerBuffer));

++             api->SetInterleavedBuffer(qToUnderlying(sourceId), buf.data(), numberOfChannels,

++                                       framesPerBuffer);

++         } else {

++             api->SetInterleavedBuffer(qToUnderlying(sourceId), nullBuffer.data(), numberOfChannels,

++                                       framesPerBuffer);

++         }

++     }

++ }

++

++ void QResonanceAudioPlayer::processSliceProcessSurroundMode(QSpan<float> outputBuffer)

++ {

++     constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);

++     using QtMultimediaPrivate::take;

++     namespace ranges = QtMultimediaPrivate::ranges;

++     Q_ASSERT(outputBuffer.size() == m_format.channelCount() * framesPerBuffer);

++

++     std::array<const float *, QAmbisonicDecoder::maxAmbisonicChannels> channels;

++     std::array<const float *, 2> reverbBuffers{};

++     int nFrames = m_resonanceAudio->getAmbisonicOutput(channels.data(), reverbBuffers.data(),

++                                                        m_ambisonicDecoder.nInputChannels());

++

++     if (nFrames < 0) {

++         // If we get here, it means that resonanceAudio did not actually fill the buffer.

++         // Sometimes this is expected, for example if resonanceAudio does not have any sources.

++         // In this case we just fill the buffer with silence.

++         ranges::fill(outputBuffer, 0);

++         return;

++     }

++     auto nSamples = m_ambisonicDecoder.outputSamples(nFrames);

++

++     Q_ASSERT(m_ambisonicDecoder.nOutputChannels() <= 8);

++     QSpan currentOutput = take(outputBuffer, nSamples);

++     m_ambisonicDecoder.processBufferWithReverb(

++             QSpan{ channels.data(), m_ambisonicDecoder.nInputChannels() }, reverbBuffers,

++             currentOutput);

++ }

++

++ void QResonanceAudioPlayer::processSliceProcessWithoutReverb(QSpan<float> outputBuffer)

++ {

++     constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);

++

++     Q_ASSERT(outputBuffer.size() == m_format.channelCount() * framesPerBuffer);

++     namespace ranges = QtMultimediaPrivate::ranges;

++

++     bool ok = m_resonanceAudio->api->FillInterleavedOutputBuffer(

++             m_format.channelCount(), framesPerBuffer, outputBuffer.data());

++     if (!ok) {

++         // If we get here, it means that resonanceAudio did not actually fill the buffer.

++         // Sometimes this is expected, for example if resonanceAudio does not have any sources.

++         // In this case we just fill the buffer with silence.

++         if (m_sounds.empty()) {

++             ranges::fill(outputBuffer, 0.f);

++         } else {

++             // If we get here, it means that something unexpected happened, so bail.

++             qWarning() << "    processSliceProcessWithoutReverb failure!";

++             return;

++         }

++     }

++ }

++

++ ///////////////////////////////////////////////////////////////////////////////////////////////////

++

++ QAudioEngineWithPlayer::QAudioEngineWithPlayer(int sampleRate) : QAudioEnginePrivate(sampleRate)

++ {

++ }

++

++ QAudioEngineWithPlayer::~QAudioEngineWithPlayer()

++ {

++     stop();

++     m_playbackEngine = nullptr;

++ }

++

++ void QAudioEngineWithPlayer::start()

++ {

++     QAudioDevice device = m_device;

++     if (device.isNull())

++         device = QMediaDevices::defaultAudioOutput();

++

++     if (device.isNull()) {

++         qWarning() << "QAudioEngine::start() failed: No audio output device found.";

++         return;

++     }

++

++     QAudioFormat format;

++     format.setSampleFormat(QAudioFormat::Float);

++     format.setSampleRate(sampleRate());

++     format.setChannelCount(std::min(2, device.maximumChannelCount()));

++

++     m_playbackEngine = std::make_shared<QRtAudioEngine>(device, format, std::nullopt,

++                                                         QAudioEnginePrivate::framesPerBuffer);

++     m_format = format;

++

++     auto voice = std::make_shared<QResonanceAudioPlayer>(this, QRtAudioEngine::allocateVoiceId());

++

++     for (auto &[sound, state] : playbackStates)

++         voice->addSound(SourceId{ sound->sourceId }, sound->nchannels, state);

++

++     m_engineVoice = voice;

++     m_playbackEngine->play(voice);

++ }

++

++ void QAudioEngineWithPlayer::stop()

++ {

++     if (!m_engineVoice)

++         return;

++     m_playbackEngine->stop(m_engineVoice);

++     m_engineVoice = {};

++ }

++

++ void QAudioEngineWithPlayer::setPaused(bool paused)

++ {

++     if (paused == m_paused)

++         return;

++     m_paused = paused;

++     if (!m_engineVoice)

++         return;

++

++     // CHECK: can we pause the stream? it seems a bit problematic, as it would prevent us from sending

++     // rt-visitors to update the state of the player.

++     m_playbackEngine->visitVoiceRt(m_engineVoice->voiceId(),

++                                    [paused](QResonanceAudioPlayer &player) {

++         player.setPaused(paused);

++     });

++ }

++

++ bool QAudioEngineWithPlayer::isPaused() const

++ {

++     return m_paused;

++ }

++

++ void QAudioEngineWithPlayer::setOutputDevice(const QAudioDevice &device)

++ {

++     if (m_device == device)

++         return;

++     if (m_engineVoice) {

++         qWarning() << "Changing device on a running engine not implemented";

++         return;

++     }

++     m_device = device;

++     Q_Q(QAudioEngine);

++     Q_EMIT q->outputDeviceChanged();

++ }

++

++ QAudioDevice QAudioEngineWithPlayer::outputDevice() const

++ {

++     return m_device;

++ }

++

++ void QAudioEngineWithPlayer::addSound(QSpatialAudioSoundPrivate *sound)

++ {

++     playbackStates.emplace(sound, nullptr);

++

++     if (!m_engineVoice)

++         return;

++

++     m_playbackEngine->visitVoiceRt(

++             m_engineVoice->voiceId(),

++             [id = SourceId{ sound->sourceId },

++              numberOfChannels = sound->nchannels](QResonanceAudioPlayer &player) mutable {

++         player.addSound(id, numberOfChannels);

++     });

++ }

++

++ void QAudioEngineWithPlayer::removeSound(QSpatialAudioSoundPrivate *sound)

++ {

++     SharedPlaybackState oldState = std::move(playbackStates[sound]);

++     playbackStates.erase(sound);

++     if (!m_engineVoice)

++         return;

++

++     // pass oldState to the lambda to ensure that it is not freed on the real-time thread

++     m_playbackEngine->visitVoiceRt(m_engineVoice->voiceId(),

++                                    [id = SourceId{ sound->sourceId },

++                                     oldState = std::move(oldState)](QResonanceAudioPlayer &player) {

++         player.removeSound(id);

++     });

++ }

++

++ void QAudioEngineWithPlayer::setSoundPlaybackData(QSpatialAudioSoundPrivate *sound,

++                                                   SharedPlaybackState state)

++ {

++     auto it = playbackStates.find(sound);

++     Q_ASSERT(it != playbackStates.end());

++

++     SharedPlaybackState oldState = std::exchange(it->second, state);

++

++     if (!m_engineVoice)

++         return;

++

++     // pass oldState to the lambda to ensure that it is not freed on the real-time thread

++     m_playbackEngine->visitVoiceRt(

++             m_engineVoice->voiceId(),

++             [id = SourceId{ sound->sourceId }, oldState = std::move(oldState),

++              state = std::move(state)](QResonanceAudioPlayer &player) mutable {

++         player.setSoundPlaybackData(id, std::move(state));

++     });

++ }

++

++ void QAudioEngineWithPlayer::setOutputMode(QAudioEngine::OutputMode mode)

++ {

++     if (outputMode() == mode)

++         return;

++     QAudioEnginePrivate::setOutputMode(mode);

++     if (!m_engineVoice)

++         return;

++

++     m_playbackEngine->visitVoiceRt(m_engineVoice->voiceId(), [mode](QResonanceAudioPlayer &player) {

++         player.setOutputMode(mode);

++     });

++ }

++

++ void QAudioEngineWithPlayer::updateRoomEffects()

++ {

++     for (auto [sound, key] : playbackStates)

++         sound->updateRoomEffects();

++ }

++

++ QT_END_NAMESPACE

Revision control