Revision control
++ // Copyright (C) 2021 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ #include "qffmpegaudioframeconverter_p.h"
++
++ #include <QtMultimedia/private/qaudiobuffer_support_p.h>
++ #include <QtFFmpegMediaPluginImpl/private/qffmpegframe_p.h>
++ #include <QtFFmpegMediaPluginImpl/private/qffmpegresampler_p.h>
++ #include <QtFFmpegMediaPluginImpl/private/qffmpegmediaformatinfo_p.h>
++
++ #if defined(Q_CC_MSVC) && defined(QT_MM_OPTIMIZE_DEBUG)
++ # pragma optimize("", on)
++ #endif
++
++ // TODO: namespace 3p library to prevent odr violations
++ #include <signalsmith-stretch.h>
++
++ QT_BEGIN_NAMESPACE
++
++ namespace QFFmpeg {
++
++ namespace {
++
++ qreal sampleRateFactor()
++ {
++ // Test purposes:
++ //
++ // The env var describes a factor for the sample rate of
++ // audio data that we feed to the audio sink.
++ //
++ // In some cases audio sink might consume data slightly slower or faster than expected;
++ // even though the synchronization in the audio renderer is supposed to handle it,
++ // it makes sense to experiment with QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR != 1.
++ //
++ // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR > 1 (e.g. 1.01 - 1.1) to test high buffer
++ // loading
++ // or try compensating too fast data consumption by the audio sink.
++ // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR < 1 to test low buffer loading
++ // or try compensating too slow data consumption by the audio sink.
++
++ static const qreal result = []() {
++ const auto sampleRateFactorStr =
++ qEnvironmentVariable("QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR");
++ bool ok = false;
++ const auto result = sampleRateFactorStr.toDouble(&ok);
++ return ok ? result : 1.;
++ }();
++
++ return result;
++ }
++
++ struct TrivialAudioFrameConverter : AbstractAudioFrameConverter
++ {
++ explicit TrivialAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat,
++ float playbackRate)
++ {
++ int sampleRate = qRound(outputFormat.sampleRate() / playbackRate * sampleRateFactor());
++ outputFormat.setSampleRate(sampleRate);
++ m_converter = createResampler(frame, outputFormat);
++ }
++
++ QAudioBuffer convert(AVFrame *frame) override
++ {
++ if (!m_converter)
++ return { };
++
++ return m_converter->resample(frame);
++ }
++
++ private:
++ std::unique_ptr<QFFmpegResampler> m_converter;
++ };
++
++ struct PitchShiftingAudioFrameConverter : AbstractAudioFrameConverter
++ {
++ explicit PitchShiftingAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat,
++ float playbackRate)
++ : m_playbackRate{ playbackRate }
++ {
++ const QAudioFormat mediaFormat = QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(
++ *frame.codecContext()->stream()->codecpar);
++
++ const QAudioFormat floatFormat = [&] {
++ QAudioFormat ret = mediaFormat;
++ ret.setSampleFormat(QAudioFormat::SampleFormat::Float);
++ return ret;
++ }();
++
++ m_toPCMDecoder = createResampler(frame, floatFormat); // this is *not* a resampler
++ m_stretcher.reset();
++ m_pendingFractionalFrames = 0.f;
++ m_stretcher.presetDefault(mediaFormat.channelCount(), outputFormat.sampleRate());
++
++ const QAudioFormat pitchCompensatedFormat = [&] {
++ QAudioFormat ret = floatFormat;
++ ret.setSampleRate(outputFormat.sampleRate());
++ return ret;
++ }();
++ m_toOutputFormatConverter = QFFmpegResampler::createFromInputFormat(pitchCompensatedFormat, outputFormat,
++ frame.startTime().get());
++ }
++
++ QAudioBuffer convert(AVFrame *frame) override
++ {
++ if (!m_toPCMDecoder || !m_toOutputFormatConverter)
++ return { };
++
++ using namespace QtPrivate;
++
++ // convert to pcm buffer
++ QAudioBuffer wordConverted = m_toPCMDecoder->resample(frame);
++
++ // compute stretch amount
++ int mediaFrameCount = wordConverted.frameCount();
++ float expectedNumberOfFrames = mediaFrameCount / m_playbackRate + m_pendingFractionalFrames;
++ int numberOfFullExpectedFrames = qFloor(expectedNumberOfFrames);
++ m_pendingFractionalFrames = expectedNumberOfFrames - numberOfFullExpectedFrames;
++
++ auto timeStretcherOutput = QAudioBuffer{
++ numberOfFullExpectedFrames,
++ wordConverted.format(),
++ };
++
++ // stretch
++ m_stretcher.process(
++ QAudioBufferDeinterleaveAdaptor<const float>{
++ wordConverted,
++ },
++ mediaFrameCount,
++ QAudioBufferDeinterleaveAdaptor<float>{
++ timeStretcherOutput,
++ },
++ numberOfFullExpectedFrames);
++
++ // convert to audio output format
++ QAudioBuffer outputBuffer = m_toOutputFormatConverter->resample(
++ timeStretcherOutput.constData<const char>(), timeStretcherOutput.byteCount());
++
++ return outputBuffer;
++ }
++
++ private:
++ std::unique_ptr<QFFmpegResampler> m_toPCMDecoder;
++ signalsmith::stretch::SignalsmithStretch<float> m_stretcher;
++ std::unique_ptr<QFFmpegResampler> m_toOutputFormatConverter;
++ float m_playbackRate;
++ float m_pendingFractionalFrames = 0.f;
++ };
++
++ } // namespace
++
++ AbstractAudioFrameConverter::~AbstractAudioFrameConverter() = default;
++
++ std::unique_ptr<QFFmpegResampler> createResampler(const Frame &frame,
++ const QAudioFormat &outputFormat)
++ {
++ return QFFmpegResampler::createFromCodecContext(frame.codecContext(), outputFormat, frame.startTime().get());
++ }
++
++ std::unique_ptr<AbstractAudioFrameConverter>
++ makeTrivialAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat, float playbackRate)
++ {
++ return std::make_unique<TrivialAudioFrameConverter>(frame, outputFormat, playbackRate);
++ }
++
++ std::unique_ptr<AbstractAudioFrameConverter>
++ makePitchShiftingAudioFrameConverter(const Frame &frame, QAudioFormat outputFormat,
++ float playbackRate)
++ {
++ return std::make_unique<PitchShiftingAudioFrameConverter>(frame, outputFormat, playbackRate);
++ }
++
++ } // namespace QFFmpeg
++
++ QT_END_NAMESPACE