Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "playbackengine/qffmpegaudiorenderer_p.h"
#include <QtMultimedia/qaudiosink.h>
#include <QtMultimedia/qaudiooutput.h>
#include <QtMultimedia/qaudiobufferoutput.h>
#include <QtMultimedia/private/qaudiobuffer_support_p.h>
#include <QtMultimedia/private/qplatformaudiooutput_p.h>
#include <QtCore/qloggingcategory.h>
#include "qffmpegaudioframeconverter_p.h"
#include "qffmpegmediaformatinfo_p.h"
#include "qffmpegresampler_p.h"
QT_BEGIN_NAMESPACE
Q_STATIC_LOGGING_CATEGORY(qLcAudioRenderer, "qt.multimedia.ffmpeg.audiorenderer");
namespace QFFmpeg {
using namespace std::chrono_literals;
using namespace std::chrono;
namespace {
constexpr auto DesiredBufferTime = 110000us;
constexpr auto MinDesiredBufferTime = 22000us;
constexpr auto MaxDesiredBufferTime = 64000us;
constexpr auto MinDesiredFreeBufferTime = 10000us;
// It might be changed with #ifdef, as on Linux, QPulseAudioSink has quite unstable timings,
// and it needs much more time to make sure that the buffer is overloaded.
constexpr auto BufferLoadingMeasureTime = 400ms;
constexpr auto DurationBias = 2ms; // avoids extra timer events
QAudioFormat audioFormatFromFrame(const Frame &frame)
{
return QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(
*frame.codecContext()->stream()->codecpar);
}
} // namespace
AudioRenderer::AudioRenderer(const PlaybackEngineObjectID &id, const TimeController &tc,
QAudioOutput *output, QAudioBufferOutput *bufferOutput,
bool pitchCompensation)
: Renderer(id, tc),
m_output(output),
m_bufferOutput(bufferOutput),
m_pitchCompensation(pitchCompensation)
{
if (output) {
// TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294
connect(output, &QAudioOutput::deviceChanged, this, &AudioRenderer::onDeviceChanged);
connect(output, &QAudioOutput::volumeChanged, this, &AudioRenderer::updateVolume);
connect(output, &QAudioOutput::mutedChanged, this, &AudioRenderer::updateVolume);
}
}
void AudioRenderer::setOutput(QAudioOutput *output)
{
setOutputInternal(m_output, output, [this](QAudioOutput *) { onDeviceChanged(); });
}
void AudioRenderer::setOutput(QAudioBufferOutput *bufferOutput)
{
setOutputInternal(m_bufferOutput, bufferOutput,
[this](QAudioBufferOutput *) { m_bufferOutputChanged = true; });
}
void AudioRenderer::setPitchCompensation(bool enabled)
{
invokePriorityMethod([this, enabled] {
if (m_pitchCompensation == enabled)
return;
m_pitchCompensation = enabled;
m_audioFrameConverter.reset();
});
}
AudioRenderer::~AudioRenderer()
{
freeOutput();
}
void AudioRenderer::updateVolume()
{
if (m_sink)
m_sink->setVolume(m_output->isMuted() ? 0.f : m_output->volume());
}
void AudioRenderer::onDeviceChanged()
{
m_deviceChanged = true;
}
void AudioRenderer::seekInternal()
{
// TODO: play with what to clean: we may find better config.
// If we reset sink and converters, move m_ioDevice, m_ioDevice,
// m_audioFrameConverter, m_bufferOutputResampler to m_sessionCtx.
constexpr bool shouldResetSink = true;
constexpr bool shouldResetConverters = true;
if constexpr (shouldResetSink) {
if (m_sink)
m_sink->reset();
m_ioDevice = nullptr;
m_bufferLoadingInfo = {};
}
if constexpr (shouldResetConverters) {
m_audioFrameConverter.reset();
m_bufferOutputResampler.reset();
}
// change AudioRenderer caches
m_sessionCtx = {};
// don't touch m_deviceChanged here
}
Renderer::RenderingResult AudioRenderer::renderInternal(Frame frame)
{
if (frame.isValid())
updateOutputs(frame);
// push to sink first in order not to waste time on resampling
// for QAudioBufferOutput
const RenderingResult result = pushFrameToOutput(frame);
if (m_sessionCtx.lastFramePushDone)
pushFrameToBufferOutput(frame);
// else // skip pushing the same data to QAudioBufferOutput
m_sessionCtx.lastFramePushDone = result.done;
return result;
}
AudioRenderer::RenderingResult AudioRenderer::pushFrameToOutput(const Frame &frame)
{
if (!m_ioDevice || !m_audioFrameConverter)
return {};
Q_ASSERT(m_sink);
auto firstFrameFlagGuard = qScopeGuard([&]() { m_firstFrameToSink = false; });
const SynchronizationStamp syncStamp{ m_sink->state(), m_sink->bytesFree(),
m_sessionCtx.bufferedData.offset, SteadyClock::now() };
if (!m_sessionCtx.bufferedData.isValid()) {
if (!frame.isValid()) {
if (std::exchange(m_sessionCtx.drained, true))
return {};
const auto time = bufferLoadingTime(syncStamp);
qCDebug(qLcAudioRenderer) << "Draining AudioRenderer, time:" << time;
return { time.count() == 0, time };
}
m_sessionCtx.bufferedData = {
m_audioFrameConverter->convert(frame.avFrame()),
};
}
if (m_sessionCtx.bufferedData.isValid()) {
// synchronize after "QIODevice::write" to deliver audio data to the sink ASAP.
auto syncGuard = qScopeGuard([&]() { updateSynchronization(syncStamp, frame); });
const auto bytesWritten = m_ioDevice->write(m_sessionCtx.bufferedData.data(),
m_sessionCtx.bufferedData.size());
m_sessionCtx.bufferedData.offset += bytesWritten;
if (m_sessionCtx.bufferedData.size() <= 0) {
m_sessionCtx.bufferedData = {};
return {};
}
const auto remainingDuration = durationForBytes(m_sessionCtx.bufferedData.size());
return { false,
std::min(remainingDuration + DurationBias, m_timings.actualBufferDuration / 2) };
}
return {};
}
void AudioRenderer::pushFrameToBufferOutput(const Frame &frame)
{
if (!m_bufferOutput)
return;
if (frame.isValid()) {
Q_ASSERT(m_bufferOutputResampler);
// TODO: get buffer from m_bufferedData if resample formats are equal
QAudioBuffer buffer = m_bufferOutputResampler->resample(frame.avFrame());
Q_EMIT m_bufferOutput->audioBufferReceived(buffer);
} else {
Q_EMIT m_bufferOutput->audioBufferReceived({});
}
}
void AudioRenderer::onPlaybackRateChanged()
{
m_audioFrameConverter.reset();
}
AudioRenderer::TimePoint AudioRenderer::nextTimePoint() const
{
const TimePoint timePoint = Renderer::nextTimePoint();
// if the first frame is expected, don't force the immediate job
if (m_firstFrameToSink)
return timePoint;
// if the sink is active, don't force the immediate job
if (!m_sink || m_sink->state() != QAudio::IdleState)
return timePoint;
// if the waiting interval is out of a heuristic fixable range,
// don't force the immediate job.
constexpr auto MaxFixableInterval = 50ms;
if (timePoint == TimePoint::min() ||
timePoint - std::chrono::steady_clock::now() > MaxFixableInterval)
return timePoint;
return TimePoint::min(); // do the job now
}
void AudioRenderer::onPauseChanged()
{
m_firstFrameToSink = true;
Renderer::onPauseChanged();
}
void AudioRenderer::initAudioFrameConverter(const Frame &frame)
{
// We recreate the frame converter whenever format or playback rate is changed
if (!m_pitchCompensation || QtPrivate::fuzzyCompare(playbackRate(), 1.0f)) {
m_audioFrameConverter = makeTrivialAudioFrameConverter(frame, m_sinkFormat, playbackRate());
} else {
m_audioFrameConverter =
makePitchShiftingAudioFrameConverter(frame, m_sinkFormat, playbackRate());
}
}
void AudioRenderer::freeOutput()
{
qCDebug(qLcAudioRenderer) << "Free audio output";
if (m_sink) {
m_sink->reset();
// TODO: inestigate if it's enough to reset the sink without deleting
m_sink.reset();
}
m_ioDevice = nullptr;
m_sessionCtx.bufferedData = {};
m_deviceChanged = false;
m_sinkFormat = {};
m_timings = {};
m_bufferLoadingInfo = {};
}
void AudioRenderer::updateOutputs(const Frame &frame)
{
if (m_deviceChanged) {
freeOutput();
m_audioFrameConverter.reset();
}
if (m_bufferOutput) {
if (m_bufferOutputChanged) {
m_bufferOutputChanged = false;
m_bufferOutputResampler.reset();
}
if (!m_bufferOutputResampler) {
QAudioFormat outputFormat = m_bufferOutput->format();
if (!outputFormat.isValid())
outputFormat = audioFormatFromFrame(frame);
m_bufferOutputResampler = createResampler(frame, outputFormat);
}
}
if (!m_output)
return;
if (!m_sinkFormat.isValid()) {
m_sinkFormat = audioFormatFromFrame(frame);
m_sinkFormat.setChannelConfig(m_output->device().channelConfiguration());
}
if (!m_sink) {
// Insert a delay here to test time offset synchronization, e.g. QThread::sleep(1)
m_sink = std::make_unique<QAudioSink>(m_output->device(), m_sinkFormat);
updateVolume();
m_sink->setBufferSize(m_sinkFormat.bytesForDuration(DesiredBufferTime.count()));
connect(m_sink.get(), &QAudioSink::stateChanged, this,
&AudioRenderer::onAudioSinkStateChanged);
m_timings.actualBufferDuration = durationForBytes(m_sink->bufferSize());
m_timings.maxSoundDelay = qMin(MaxDesiredBufferTime,
m_timings.actualBufferDuration - MinDesiredFreeBufferTime);
m_timings.minSoundDelay = MinDesiredBufferTime;
Q_ASSERT(DurationBias < m_timings.minSoundDelay
&& m_timings.maxSoundDelay < m_timings.actualBufferDuration);
}
if (!m_ioDevice) {
m_ioDevice = m_sink->start();
m_firstFrameToSink = true;
}
if (!m_audioFrameConverter)
initAudioFrameConverter(frame);
}
void AudioRenderer::updateSynchronization(const SynchronizationStamp &stamp, const Frame &frame)
{
if (!frame.isValid())
return;
Q_ASSERT(m_sink);
const auto bufferLoadingTime = this->bufferLoadingTime(stamp);
const auto currentFrameDelay = frameDelay(frame, stamp.timePoint);
const auto writtenTime = durationForBytes(stamp.bufferBytesWritten);
const auto soundDelay = currentFrameDelay + bufferLoadingTime - writtenTime;
auto synchronize = [&](microseconds fixedDelay, microseconds targetSoundDelay) {
// TODO: investigate if we need sample compensation here
changeRendererTime(fixedDelay - targetSoundDelay);
if (qLcAudioRenderer().isDebugEnabled()) {
// clang-format off
qCDebug(qLcAudioRenderer)
<< "Change rendering time:"
<< "\n First frame:" << m_firstFrameToSink
<< "\n Delay (frame+buffer-written):" << currentFrameDelay << "+"
<< bufferLoadingTime << "-"
<< writtenTime << "="
<< soundDelay
<< "\n Fixed delay:" << fixedDelay
<< "\n Target delay:" << targetSoundDelay
<< "\n Buffer durations (min/max/limit):" << m_timings.minSoundDelay
<< m_timings.maxSoundDelay
<< m_timings.actualBufferDuration
<< "\n Audio sink state:" << stamp.audioSinkState;
// clang-format on
}
};
const auto loadingType = soundDelay > m_timings.maxSoundDelay ? BufferLoadingInfo::High
: soundDelay < m_timings.minSoundDelay ? BufferLoadingInfo::Low
: BufferLoadingInfo::Moderate;
if (loadingType != m_bufferLoadingInfo.type) {
// qCDebug(qLcAudioRenderer) << "Change buffer loading type:" <<
// m_bufferLoadingInfo.type
// << "->" << loadingType << "soundDelay:" << soundDelay;
m_bufferLoadingInfo = { loadingType, stamp.timePoint, soundDelay };
}
if (loadingType != BufferLoadingInfo::Moderate) {
const auto isHigh = loadingType == BufferLoadingInfo::High;
const auto shouldHandleIdle = stamp.audioSinkState == QAudio::IdleState && !isHigh;
auto &fixedDelay = m_bufferLoadingInfo.delay;
fixedDelay = shouldHandleIdle ? soundDelay
: isHigh ? qMin(soundDelay, fixedDelay)
: qMax(soundDelay, fixedDelay);
if (stamp.timePoint - m_bufferLoadingInfo.timePoint > BufferLoadingMeasureTime
|| (m_firstFrameToSink && isHigh) || shouldHandleIdle) {
const auto targetDelay = isHigh
? (m_timings.maxSoundDelay + m_timings.minSoundDelay) / 2
: m_timings.minSoundDelay + DurationBias;
synchronize(fixedDelay, targetDelay);
m_bufferLoadingInfo = { BufferLoadingInfo::Moderate, stamp.timePoint, targetDelay };
}
}
}
microseconds AudioRenderer::bufferLoadingTime(const SynchronizationStamp &syncStamp) const
{
Q_ASSERT(m_sink);
if (syncStamp.audioSinkState == QAudio::IdleState)
return microseconds(0);
const auto bytes = qMax(m_sink->bufferSize() - syncStamp.audioSinkBytesFree, 0);
return durationForBytes(bytes);
}
void AudioRenderer::onAudioSinkStateChanged(QAudio::State state)
{
if (state == QAudio::IdleState && !m_firstFrameToSink && !m_deviceChanged)
scheduleNextStep();
}
microseconds AudioRenderer::durationForBytes(qsizetype bytes) const
{
return microseconds(m_sinkFormat.durationForBytes(static_cast<qint32>(bytes)));
}
} // namespace QFFmpeg
QT_END_NAMESPACE
#include "moc_qffmpegaudiorenderer_p.cpp"