Revision control
++ // Copyright (C) 2021 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ #include "playbackengine/qffmpegaudiorenderer_p.h"
++
++ #include <QtMultimedia/qaudiosink.h>
++ #include <QtMultimedia/qaudiooutput.h>
++ #include <QtMultimedia/qaudiobufferoutput.h>
++ #include <QtMultimedia/private/qaudiobuffer_support_p.h>
++ #include <QtMultimedia/private/qplatformaudiooutput_p.h>
++
++ #include <QtCore/qloggingcategory.h>
++
++ #include "qffmpegaudioframeconverter_p.h"
++ #include "qffmpegmediaformatinfo_p.h"
++ #include "qffmpegresampler_p.h"
++
++ QT_BEGIN_NAMESPACE
++
++ Q_STATIC_LOGGING_CATEGORY(qLcAudioRenderer, "qt.multimedia.ffmpeg.audiorenderer");
++
++ namespace QFFmpeg {
++
++ using namespace std::chrono_literals;
++ using namespace std::chrono;
++
++ namespace {
++ constexpr auto DesiredBufferTime = 110000us;
++ constexpr auto MinDesiredBufferTime = 22000us;
++ constexpr auto MaxDesiredBufferTime = 64000us;
++ constexpr auto MinDesiredFreeBufferTime = 10000us;
++
++ // It might be changed with #ifdef, as on Linux, QPulseAudioSink has quite unstable timings,
++ // and it needs much more time to make sure that the buffer is overloaded.
++ constexpr auto BufferLoadingMeasureTime = 400ms;
++
++ constexpr auto DurationBias = 2ms; // avoids extra timer events
++
++ QAudioFormat audioFormatFromFrame(const Frame &frame)
++ {
++ return QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(
++ *frame.codecContext()->stream()->codecpar);
++ }
++
++ } // namespace
++
++ AudioRenderer::AudioRenderer(const PlaybackEngineObjectID &id, const TimeController &tc,
++ QAudioOutput *output, QAudioBufferOutput *bufferOutput,
++ bool pitchCompensation)
++ : Renderer(id, tc),
++ m_output(output),
++ m_bufferOutput(bufferOutput),
++ m_pitchCompensation(pitchCompensation)
++ {
++ if (output) {
++ // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294
++ connect(output, &QAudioOutput::deviceChanged, this, &AudioRenderer::onDeviceChanged);
++ connect(output, &QAudioOutput::volumeChanged, this, &AudioRenderer::updateVolume);
++ connect(output, &QAudioOutput::mutedChanged, this, &AudioRenderer::updateVolume);
++ }
++ }
++
++ void AudioRenderer::setOutput(QAudioOutput *output)
++ {
++ setOutputInternal(m_output, output, [this](QAudioOutput *) { onDeviceChanged(); });
++ }
++
++ void AudioRenderer::setOutput(QAudioBufferOutput *bufferOutput)
++ {
++ setOutputInternal(m_bufferOutput, bufferOutput,
++ [this](QAudioBufferOutput *) { m_bufferOutputChanged = true; });
++ }
++
++ void AudioRenderer::setPitchCompensation(bool enabled)
++ {
++ invokePriorityMethod([this, enabled] {
++ if (m_pitchCompensation == enabled)
++ return;
++
++ m_pitchCompensation = enabled;
++ m_audioFrameConverter.reset();
++ });
++ }
++
++ AudioRenderer::~AudioRenderer()
++ {
++ freeOutput();
++ }
++
++ void AudioRenderer::updateVolume()
++ {
++ if (m_sink)
++ m_sink->setVolume(m_output->isMuted() ? 0.f : m_output->volume());
++ }
++
++ void AudioRenderer::onDeviceChanged()
++ {
++ m_deviceChanged = true;
++ }
++
++ void AudioRenderer::seekInternal()
++ {
++ // TODO: play with what to clean: we may find better config.
++ // If we reset sink and converters, move m_ioDevice, m_ioDevice,
++ // m_audioFrameConverter, m_bufferOutputResampler to m_sessionCtx.
++ constexpr bool shouldResetSink = true;
++ constexpr bool shouldResetConverters = true;
++
++ if constexpr (shouldResetSink) {
++ if (m_sink)
++ m_sink->reset();
++ m_ioDevice = nullptr;
++ m_bufferLoadingInfo = {};
++ }
++
++ if constexpr (shouldResetConverters) {
++ m_audioFrameConverter.reset();
++ m_bufferOutputResampler.reset();
++ }
++
++ // change AudioRenderer caches
++ m_sessionCtx = {};
++ // don't touch m_deviceChanged here
++ }
++
++ Renderer::RenderingResult AudioRenderer::renderInternal(Frame frame)
++ {
++ if (frame.isValid())
++ updateOutputs(frame);
++
++ // push to sink first in order not to waste time on resampling
++ // for QAudioBufferOutput
++ const RenderingResult result = pushFrameToOutput(frame);
++
++ if (m_sessionCtx.lastFramePushDone)
++ pushFrameToBufferOutput(frame);
++ // else // skip pushing the same data to QAudioBufferOutput
++
++ m_sessionCtx.lastFramePushDone = result.done;
++
++ return result;
++ }
++
++ AudioRenderer::RenderingResult AudioRenderer::pushFrameToOutput(const Frame &frame)
++ {
++ if (!m_ioDevice || !m_audioFrameConverter)
++ return {};
++
++ Q_ASSERT(m_sink);
++
++ auto firstFrameFlagGuard = qScopeGuard([&]() { m_firstFrameToSink = false; });
++
++ const SynchronizationStamp syncStamp{ m_sink->state(), m_sink->bytesFree(),
++ m_sessionCtx.bufferedData.offset, SteadyClock::now() };
++
++ if (!m_sessionCtx.bufferedData.isValid()) {
++ if (!frame.isValid()) {
++ if (std::exchange(m_sessionCtx.drained, true))
++ return {};
++
++ const auto time = bufferLoadingTime(syncStamp);
++
++ qCDebug(qLcAudioRenderer) << "Draining AudioRenderer, time:" << time;
++
++ return { time.count() == 0, time };
++ }
++
++ m_sessionCtx.bufferedData = {
++ m_audioFrameConverter->convert(frame.avFrame()),
++ };
++ }
++
++ if (m_sessionCtx.bufferedData.isValid()) {
++ // synchronize after "QIODevice::write" to deliver audio data to the sink ASAP.
++ auto syncGuard = qScopeGuard([&]() { updateSynchronization(syncStamp, frame); });
++
++ const auto bytesWritten = m_ioDevice->write(m_sessionCtx.bufferedData.data(),
++ m_sessionCtx.bufferedData.size());
++
++ m_sessionCtx.bufferedData.offset += bytesWritten;
++
++ if (m_sessionCtx.bufferedData.size() <= 0) {
++ m_sessionCtx.bufferedData = {};
++
++ return {};
++ }
++
++ const auto remainingDuration = durationForBytes(m_sessionCtx.bufferedData.size());
++
++ return { false,
++ std::min(remainingDuration + DurationBias, m_timings.actualBufferDuration / 2) };
++ }
++
++ return {};
++ }
++
++ void AudioRenderer::pushFrameToBufferOutput(const Frame &frame)
++ {
++ if (!m_bufferOutput)
++ return;
++
++ if (frame.isValid()) {
++ Q_ASSERT(m_bufferOutputResampler);
++
++ // TODO: get buffer from m_bufferedData if resample formats are equal
++ QAudioBuffer buffer = m_bufferOutputResampler->resample(frame.avFrame());
++ Q_EMIT m_bufferOutput->audioBufferReceived(buffer);
++ } else {
++ Q_EMIT m_bufferOutput->audioBufferReceived({});
++ }
++ }
++
++ void AudioRenderer::onPlaybackRateChanged()
++ {
++ m_audioFrameConverter.reset();
++ }
++
++ AudioRenderer::TimePoint AudioRenderer::nextTimePoint() const
++ {
++ const TimePoint timePoint = Renderer::nextTimePoint();
++
++ // if the first frame is expected, don't force the immediate job
++ if (m_firstFrameToSink)
++ return timePoint;
++
++ // if the sink is active, don't force the immediate job
++ if (!m_sink || m_sink->state() != QAudio::IdleState)
++ return timePoint;
++
++ // if the waiting interval is out of a heuristic fixable range,
++ // don't force the immediate job.
++ constexpr auto MaxFixableInterval = 50ms;
++ if (timePoint == TimePoint::min() ||
++ timePoint - std::chrono::steady_clock::now() > MaxFixableInterval)
++ return timePoint;
++
++ return TimePoint::min(); // do the job now
++ }
++
++ void AudioRenderer::onPauseChanged()
++ {
++ m_firstFrameToSink = true;
++ Renderer::onPauseChanged();
++ }
++
++ void AudioRenderer::initAudioFrameConverter(const Frame &frame)
++ {
++ // We recreate the frame converter whenever format or playback rate is changed
++ if (!m_pitchCompensation || QtPrivate::fuzzyCompare(playbackRate(), 1.0f)) {
++ m_audioFrameConverter = makeTrivialAudioFrameConverter(frame, m_sinkFormat, playbackRate());
++ } else {
++ m_audioFrameConverter =
++ makePitchShiftingAudioFrameConverter(frame, m_sinkFormat, playbackRate());
++ }
++ }
++
++ void AudioRenderer::freeOutput()
++ {
++ qCDebug(qLcAudioRenderer) << "Free audio output";
++ if (m_sink) {
++ m_sink->reset();
++
++ // TODO: inestigate if it's enough to reset the sink without deleting
++ m_sink.reset();
++ }
++
++ m_ioDevice = nullptr;
++
++ m_sessionCtx.bufferedData = {};
++ m_deviceChanged = false;
++ m_sinkFormat = {};
++ m_timings = {};
++ m_bufferLoadingInfo = {};
++ }
++
++ void AudioRenderer::updateOutputs(const Frame &frame)
++ {
++ if (m_deviceChanged) {
++ freeOutput();
++ m_audioFrameConverter.reset();
++ }
++
++ if (m_bufferOutput) {
++ if (m_bufferOutputChanged) {
++ m_bufferOutputChanged = false;
++ m_bufferOutputResampler.reset();
++ }
++
++ if (!m_bufferOutputResampler) {
++ QAudioFormat outputFormat = m_bufferOutput->format();
++ if (!outputFormat.isValid())
++ outputFormat = audioFormatFromFrame(frame);
++ m_bufferOutputResampler = createResampler(frame, outputFormat);
++ }
++ }
++
++ if (!m_output)
++ return;
++
++ if (!m_sinkFormat.isValid()) {
++ m_sinkFormat = audioFormatFromFrame(frame);
++ m_sinkFormat.setChannelConfig(m_output->device().channelConfiguration());
++ }
++
++ if (!m_sink) {
++ // Insert a delay here to test time offset synchronization, e.g. QThread::sleep(1)
++ m_sink = std::make_unique<QAudioSink>(m_output->device(), m_sinkFormat);
++ updateVolume();
++ m_sink->setBufferSize(m_sinkFormat.bytesForDuration(DesiredBufferTime.count()));
++
++ connect(m_sink.get(), &QAudioSink::stateChanged, this,
++ &AudioRenderer::onAudioSinkStateChanged);
++
++ m_timings.actualBufferDuration = durationForBytes(m_sink->bufferSize());
++ m_timings.maxSoundDelay = qMin(MaxDesiredBufferTime,
++ m_timings.actualBufferDuration - MinDesiredFreeBufferTime);
++ m_timings.minSoundDelay = MinDesiredBufferTime;
++
++ Q_ASSERT(DurationBias < m_timings.minSoundDelay
++ && m_timings.maxSoundDelay < m_timings.actualBufferDuration);
++ }
++
++ if (!m_ioDevice) {
++ m_ioDevice = m_sink->start();
++ m_firstFrameToSink = true;
++ }
++
++ if (!m_audioFrameConverter)
++ initAudioFrameConverter(frame);
++ }
++
++ void AudioRenderer::updateSynchronization(const SynchronizationStamp &stamp, const Frame &frame)
++ {
++ if (!frame.isValid())
++ return;
++
++ Q_ASSERT(m_sink);
++
++ const auto bufferLoadingTime = this->bufferLoadingTime(stamp);
++ const auto currentFrameDelay = frameDelay(frame, stamp.timePoint);
++ const auto writtenTime = durationForBytes(stamp.bufferBytesWritten);
++ const auto soundDelay = currentFrameDelay + bufferLoadingTime - writtenTime;
++
++ auto synchronize = [&](microseconds fixedDelay, microseconds targetSoundDelay) {
++ // TODO: investigate if we need sample compensation here
++
++ changeRendererTime(fixedDelay - targetSoundDelay);
++ if (qLcAudioRenderer().isDebugEnabled()) {
++ // clang-format off
++ qCDebug(qLcAudioRenderer)
++ << "Change rendering time:"
++ << "\n First frame:" << m_firstFrameToSink
++ << "\n Delay (frame+buffer-written):" << currentFrameDelay << "+"
++ << bufferLoadingTime << "-"
++ << writtenTime << "="
++ << soundDelay
++ << "\n Fixed delay:" << fixedDelay
++ << "\n Target delay:" << targetSoundDelay
++ << "\n Buffer durations (min/max/limit):" << m_timings.minSoundDelay
++ << m_timings.maxSoundDelay
++ << m_timings.actualBufferDuration
++ << "\n Audio sink state:" << stamp.audioSinkState;
++ // clang-format on
++ }
++ };
++
++ const auto loadingType = soundDelay > m_timings.maxSoundDelay ? BufferLoadingInfo::High
++ : soundDelay < m_timings.minSoundDelay ? BufferLoadingInfo::Low
++ : BufferLoadingInfo::Moderate;
++
++ if (loadingType != m_bufferLoadingInfo.type) {
++ // qCDebug(qLcAudioRenderer) << "Change buffer loading type:" <<
++ // m_bufferLoadingInfo.type
++ // << "->" << loadingType << "soundDelay:" << soundDelay;
++ m_bufferLoadingInfo = { loadingType, stamp.timePoint, soundDelay };
++ }
++
++ if (loadingType != BufferLoadingInfo::Moderate) {
++ const auto isHigh = loadingType == BufferLoadingInfo::High;
++ const auto shouldHandleIdle = stamp.audioSinkState == QAudio::IdleState && !isHigh;
++
++ auto &fixedDelay = m_bufferLoadingInfo.delay;
++
++ fixedDelay = shouldHandleIdle ? soundDelay
++ : isHigh ? qMin(soundDelay, fixedDelay)
++ : qMax(soundDelay, fixedDelay);
++
++ if (stamp.timePoint - m_bufferLoadingInfo.timePoint > BufferLoadingMeasureTime
++ || (m_firstFrameToSink && isHigh) || shouldHandleIdle) {
++ const auto targetDelay = isHigh
++ ? (m_timings.maxSoundDelay + m_timings.minSoundDelay) / 2
++ : m_timings.minSoundDelay + DurationBias;
++
++ synchronize(fixedDelay, targetDelay);
++ m_bufferLoadingInfo = { BufferLoadingInfo::Moderate, stamp.timePoint, targetDelay };
++ }
++ }
++ }
++
++ microseconds AudioRenderer::bufferLoadingTime(const SynchronizationStamp &syncStamp) const
++ {
++ Q_ASSERT(m_sink);
++
++ if (syncStamp.audioSinkState == QAudio::IdleState)
++ return microseconds(0);
++
++ const auto bytes = qMax(m_sink->bufferSize() - syncStamp.audioSinkBytesFree, 0);
++ return durationForBytes(bytes);
++ }
++
++ void AudioRenderer::onAudioSinkStateChanged(QAudio::State state)
++ {
++ if (state == QAudio::IdleState && !m_firstFrameToSink && !m_deviceChanged)
++ scheduleNextStep();
++ }
++
++ microseconds AudioRenderer::durationForBytes(qsizetype bytes) const
++ {
++ return microseconds(m_sinkFormat.durationForBytes(static_cast<qint32>(bytes)));
++ }
++
++ } // namespace QFFmpeg
++
++ QT_END_NAMESPACE
++
++ #include "moc_qffmpegaudiorenderer_p.cpp"