Revision control
++ // Copyright (C) 2024 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++ #include "qffmpegaudioencoder_p.h"
++ #include "qffmpegrecordingengineutils_p.h"
++ #include "qffmpegaudioencoderutils_p.h"
++ #include "qffmpegaudioinput_p.h"
++ #include "qffmpegencoderoptions_p.h"
++ #include "qffmpegmuxer_p.h"
++ #include "qffmpegrecordingengine_p.h"
++ #include "qffmpegmediaformatinfo_p.h"
++ #include "qffmpegcodecstorage_p.h"
++ #include <QtCore/qloggingcategory.h>
++
++ QT_BEGIN_NAMESPACE
++
++ namespace QFFmpeg {
++
++ Q_STATIC_LOGGING_CATEGORY(qLcFFmpegAudioEncoder, "qt.multimedia.ffmpeg.audioencoder");
++ static constexpr bool audioEncoderExtendedTracing = false;
++
++ namespace {
++ void setupStreamParameters(AVStream *stream, const Codec &codec,
++ const AVAudioFormat &requestedAudioFormat)
++ {
++ const auto channelLayouts = codec.channelLayouts();
++ #if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
++ stream->codecpar->ch_layout =
++ adjustChannelLayout(channelLayouts, requestedAudioFormat.channelLayout);
++ #else
++ stream->codecpar->channel_layout =
++ adjustChannelLayout(channelLayouts, requestedAudioFormat.channelLayoutMask);
++ stream->codecpar->channels = qPopulationCount(stream->codecpar->channel_layout);
++ #endif
++ const auto sampleRates = codec.sampleRates();
++ const auto sampleRate = adjustSampleRate(sampleRates, requestedAudioFormat.sampleRate);
++
++ stream->codecpar->sample_rate = sampleRate;
++ // Codecs with a variable frame size (e.g. PCM) must keep frame_size at 0;
++ // FFmpeg 9 rejects any frame whose nb_samples exceeds a non-zero frame_size,
++ // even for such codecs. Codecs with a fixed frame size determine and set
++ // their own frame_size during avcodec_open2, overriding this default.
++ if (!(codec.capabilities() & AV_CODEC_CAP_VARIABLE_FRAME_SIZE))
++ stream->codecpar->frame_size = 1024;
++ const auto sampleFormats = codec.sampleFormats();
++ stream->codecpar->format = adjustSampleFormat(sampleFormats, requestedAudioFormat.sampleFormat);
++
++ stream->time_base = AVRational{ 1, sampleRate };
++
++ qCDebug(qLcFFmpegAudioEncoder)
++ << "set stream time_base" << stream->time_base.num << "/" << stream->time_base.den;
++ }
++
++ bool openCodecContext(AVCodecContext *codecContext, AVStream *stream,
++ const QMediaEncoderSettings &settings)
++ {
++ Q_ASSERT(codecContext);
++ codecContext->time_base = stream->time_base;
++
++ int status = avcodec_parameters_to_context(codecContext, stream->codecpar);
++ if (status < 0) {
++ qCWarning(qLcFFmpegAudioEncoder)
++ << "Cannot set codec parameters; result:" << AVError(status);
++ return false;
++ }
++
++ // if avcodec_open2 fails, it may clean codecContext->codec
++ Codec codec{ codecContext->codec };
++
++ AVDictionaryHolder opts;
++ applyAudioEncoderOptions(settings, codec.name(), codecContext, opts);
++ applyExperimentalCodecOptions(codec, opts);
++
++ status = avcodec_open2(codecContext, codec.get(), opts);
++
++ if (status != 0) {
++ qCWarning(qLcFFmpegAudioEncoder)
++ << "Cannot open audio codec" << codec.name() << "; result:" << AVError(status);
++ return false;
++ }
++
++ qCDebug(qLcFFmpegAudioEncoder) << "audio codec params: fmt=" << codecContext->sample_fmt
++ << "rate=" << codecContext->sample_rate;
++
++ status = avcodec_parameters_from_context(stream->codecpar, codecContext);
++ if (status < 0) {
++ qCWarning(qLcFFmpegAudioEncoder) << "Cannot get parameters from audio codec context"
++ << codec.name() << "; result:" << AVError(status);
++ return false;
++ }
++
++ return true;
++ }
++
++ } // namespace
++
++ AudioEncoder::AudioEncoder(RecordingEngine &recordingEngine, const QAudioFormat &sourceFormat,
++ const QMediaEncoderSettings &settings)
++ : EncoderThread(recordingEngine), m_sourceFormat(sourceFormat), m_settings(settings)
++ {
++ setObjectName(QLatin1String("AudioEncoder"));
++ qCDebug(qLcFFmpegAudioEncoder) << "AudioEncoder" << settings.audioCodec();
++
++ const AVCodecID codecID = QFFmpegMediaFormatInfo::codecIdForAudioCodec(settings.audioCodec());
++ Q_ASSERT(avformat_query_codec(recordingEngine.avFormatContext()->oformat, codecID,
++ FF_COMPLIANCE_NORMAL));
++
++ Q_ASSERT(QFFmpeg::findAVEncoder(codecID));
++
++ m_stream = avformat_new_stream(recordingEngine.avFormatContext(), nullptr);
++ m_stream->id = recordingEngine.avFormatContext()->nb_streams - 1;
++ m_stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
++ m_stream->codecpar->codec_id = codecID;
++ }
++
++ void AudioEncoder::addBuffer(const QAudioBuffer &buffer)
++ {
++ if (!buffer.isValid()) {
++ setEndOfSourceStream();
++ return;
++ }
++
++ {
++ const std::chrono::microseconds bufferDuration(buffer.duration());
++ auto guard = lockLoopData();
++
++ resetEndOfSourceStream();
++
++ if (m_paused)
++ return;
++
++ // TODO: apply logic with canPushFrame
++
++ m_audioBufferQueue.push(buffer);
++ m_queueDuration += bufferDuration;
++ }
++
++ dataReady();
++ }
++
++ QAudioBuffer AudioEncoder::takeBuffer()
++ {
++ auto locker = lockLoopData();
++ QAudioBuffer result = dequeueIfPossible(m_audioBufferQueue);
++ m_queueDuration -= std::chrono::microseconds(result.duration());
++ return result;
++ }
++
++ bool AudioEncoder::init()
++ {
++ const AVAudioFormat requestedAudioFormat(m_sourceFormat);
++ const AVCodecID codecId = m_stream->codecpar->codec_id;
++
++ namespace ranges = QtMultimediaPrivate::ranges;
++
++ const auto encoders = findAndScoreEncoders(codecId, [&](const Codec &codec) {
++ AVScore result = DefaultAVScore;
++
++ // Attempt to find no-conversion format
++ if (auto fmts = codec.sampleFormats(); !fmts.empty())
++ result += ranges::contains(fmts, requestedAudioFormat.sampleFormat) ? 1 : -1;
++
++ if (auto rates = codec.sampleRates(); !rates.empty())
++ result += ranges::contains(rates, requestedAudioFormat.sampleRate) ? 1 : -1;
++
++ #if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
++ if (auto layouts = codec.channelLayouts(); !layouts.empty())
++ result += ranges::contains(layouts, requestedAudioFormat.channelLayout) ? 1 : -1;
++ #else
++ if (auto layouts = codec.channelLayouts(); !layouts.empty())
++ result +=
++ ranges::contains(layouts, requestedAudioFormat.channelLayoutMask) ? 1 : -1;
++ #endif
++
++ return result;
++ });
++
++ for (const auto &[codec, score] : encoders) {
++ AVCodecContextUPtr codecContext(avcodec_alloc_context3(codec.get()));
++ if (!codecContext)
++ continue;
++
++ setupStreamParameters(m_stream, Codec{ codecContext->codec }, requestedAudioFormat);
++ if (!openCodecContext(codecContext.get(), m_stream, m_settings))
++ continue;
++
++ m_codecContext = std::move(codecContext);
++ break;
++ }
++
++ if (!m_codecContext) {
++ qCWarning(qLcFFmpegAudioEncoder) << "Unable to open any audio codec";
++ markResolved(false);
++ Q_EMIT m_recordingEngine.sessionError(QMediaRecorder::FormatError,
++ QStringLiteral("Cannot open any audio codec"));
++ return false;
++ }
++
++ qCDebug(qLcFFmpegAudioEncoder) << "found audio codec" << m_codecContext->codec->name;
++
++ updateResampler(m_sourceFormat);
++
++ // TODO: try to address this dependency here.
++ if (auto input = qobject_cast<QFFmpegAudioInput *>(source()))
++ input->setBufferSize(m_codecContext->frame_size);
++
++ return EncoderThread::init();
++ }
++
++ void AudioEncoder::cleanup()
++ {
++ while (!m_audioBufferQueue.empty())
++ processOne();
++
++ if (m_avFrameSamplesOffset) {
++ // the size of the last frame can be less than m_codecContext->frame_size
++
++ retrievePackets();
++ sendPendingFrameToAVCodec();
++ }
++
++ while (avcodec_send_frame(m_codecContext.get(), nullptr) == AVERROR(EAGAIN))
++ retrievePackets();
++ retrievePackets();
++ }
++
++ bool AudioEncoder::hasData() const
++ {
++ return !m_audioBufferQueue.empty();
++ }
++
++ void AudioEncoder::retrievePackets()
++ {
++ while (true) {
++ AVPacketUPtr packet(av_packet_alloc());
++ int ret = avcodec_receive_packet(m_codecContext.get(), packet.get());
++ switch (ret) {
++ case 0:
++ break;
++ case AVERROR(EAGAIN):
++ case AVERROR(EOF):
++ return;
++ default:
++ qCDebug(qLcFFmpegAudioEncoder) << "receive packet" << ret << QFFmpeg::AVError{ ret };
++ return;
++ }
++
++ if constexpr (audioEncoderExtendedTracing)
++ qCDebug(qLcFFmpegAudioEncoder)
++ << "writing audio packet" << packet->size << packet->pts << packet->dts;
++ packet->stream_index = m_stream->id;
++ m_recordingEngine.getMuxer()->addPacket(std::move(packet));
++ }
++ }
++
++ void AudioEncoder::processOne()
++ {
++ QAudioBuffer buffer = takeBuffer();
++ Q_ASSERT(buffer.isValid());
++
++ if constexpr (audioEncoderExtendedTracing)
++ qCDebug(qLcFFmpegAudioEncoder)
++ << "new audio buffer" << buffer.byteCount() << buffer.format()
++ << buffer.frameCount() << m_codecContext->frame_size;
++
++ if (buffer.format() != m_sourceFormat && !updateResampler(buffer.format()))
++ return;
++
++ int samplesOffset = 0;
++ const int bufferSamplesCount = static_cast<int>(buffer.frameCount());
++
++ while (samplesOffset < bufferSamplesCount)
++ handleAudioData(buffer.constData<uint8_t>(), samplesOffset, bufferSamplesCount);
++
++ Q_ASSERT(samplesOffset == bufferSamplesCount);
++ }
++
++ bool AudioEncoder::checkIfCanPushFrame() const
++ {
++ if (m_encodingStarted)
++ return m_audioBufferQueue.size() <= 1 || m_queueDuration < m_maxQueueDuration;
++ if (!isFinished())
++ return m_audioBufferQueue.empty();
++
++ return false;
++ }
++
++ bool AudioEncoder::updateResampler(const QAudioFormat &sourceFormat)
++ {
++ m_resampler.reset();
++
++ const AVAudioFormat requestedAudioFormat(sourceFormat);
++ const AVAudioFormat codecAudioFormat(m_codecContext.get());
++
++ if (requestedAudioFormat != codecAudioFormat) {
++ m_resampler = createResampleContext(requestedAudioFormat, codecAudioFormat);
++ if (!swr_is_initialized(m_resampler.get())) {
++ m_sourceFormat = {};
++ qCWarning(qLcFFmpegAudioEncoder) << "Cannot initialize resampler for audio encoder";
++ Q_EMIT m_recordingEngine.sessionError(
++ QMediaRecorder::FormatError,
++ QStringLiteral("Cannot initialize resampler for audio encoder"));
++ return false;
++ }
++ qCDebug(qLcFFmpegAudioEncoder) << "Created resampler with audio formats conversion\n"
++ << requestedAudioFormat << "->" << codecAudioFormat;
++ } else {
++ qCDebug(qLcFFmpegAudioEncoder) << "Resampler is not needed due to no-conversion format\n"
++ << requestedAudioFormat;
++ }
++
++ m_sourceFormat = sourceFormat;
++
++ return true;
++ }
++
++ void AudioEncoder::ensurePendingFrame(int availableSamplesCount)
++ {
++ Q_ASSERT(availableSamplesCount >= 0);
++
++ if (m_avFrame)
++ return;
++
++ m_avFrame = makeAVFrame();
++
++ m_avFrame->format = m_codecContext->sample_fmt;
++ #if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
++ m_avFrame->ch_layout = m_codecContext->ch_layout;
++ #else
++ m_avFrame->channel_layout = m_codecContext->channel_layout;
++ m_avFrame->channels = m_codecContext->channels;
++ #endif
++ m_avFrame->sample_rate = m_codecContext->sample_rate;
++
++ const bool isFixedFrameSize =
++ !(m_codecContext->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)
++ && m_codecContext->frame_size;
++ m_avFrame->nb_samples = isFixedFrameSize ? m_codecContext->frame_size : availableSamplesCount;
++ if (m_avFrame->nb_samples) {
++ const int status = av_frame_get_buffer(m_avFrame.get(), 0);
++ if (status < 0)
++ qCWarning(qLcFFmpegAudioEncoder)
++ << "Failed to allocate audio frame buffer:" << AVError(status);
++ }
++
++ const auto &timeBase = m_stream->time_base;
++ const auto pts = timeBase.den && timeBase.num
++ ? timeBase.den * m_samplesWritten / (m_codecContext->sample_rate * timeBase.num)
++ : m_samplesWritten;
++ setAVFrameTime(*m_avFrame, pts, timeBase);
++ }
++
++ void AudioEncoder::writeDataToPendingFrame(const uchar *data, int &samplesOffset, int samplesCount)
++ {
++ Q_ASSERT(m_avFrame);
++ Q_ASSERT(m_avFrameSamplesOffset <= m_avFrame->nb_samples);
++
++ const int bytesPerSample = av_get_bytes_per_sample(m_codecContext->sample_fmt);
++ const bool isPlanar = av_sample_fmt_is_planar(m_codecContext->sample_fmt);
++
++ #if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
++ const int channelsCount = m_codecContext->ch_layout.nb_channels;
++ #else
++ const int channelsCount = m_codecContext->channels;
++ #endif
++
++ const int audioDataOffset = isPlanar ? bytesPerSample * m_avFrameSamplesOffset
++ : bytesPerSample * m_avFrameSamplesOffset * channelsCount;
++
++ const int planesCount = isPlanar ? channelsCount : 1;
++ m_avFramePlanesData.resize(planesCount);
++ for (int plane = 0; plane < planesCount; ++plane)
++ m_avFramePlanesData[plane] = m_avFrame->extended_data[plane] + audioDataOffset;
++
++ const int samplesToWrite = m_avFrame->nb_samples - m_avFrameSamplesOffset;
++ int samplesToRead =
++ (samplesToWrite * m_sourceFormat.sampleRate() + m_codecContext->sample_rate / 2)
++ / m_codecContext->sample_rate;
++ // the lower bound is need to get round infinite loops in corner cases
++ samplesToRead = qBound(1, samplesToRead, samplesCount - samplesOffset);
++
++ data += m_sourceFormat.bytesForFrames(samplesOffset);
++
++ if (m_resampler) {
++ m_avFrameSamplesOffset += swr_convert(m_resampler.get(), m_avFramePlanesData.data(),
++ samplesToWrite, &data, samplesToRead);
++ } else {
++ Q_ASSERT(planesCount == 1);
++ m_avFrameSamplesOffset += samplesToRead;
++ memcpy(m_avFramePlanesData[0], data, m_sourceFormat.bytesForFrames(samplesToRead));
++ }
++
++ samplesOffset += samplesToRead;
++ }
++
++ void AudioEncoder::sendPendingFrameToAVCodec()
++ {
++ Q_ASSERT(m_avFrame);
++ Q_ASSERT(m_avFrameSamplesOffset <= m_avFrame->nb_samples);
++
++ m_avFrame->nb_samples = m_avFrameSamplesOffset;
++
++ m_samplesWritten += m_avFrameSamplesOffset;
++
++ const qint64 time = m_sourceFormat.durationForFrames(
++ m_samplesWritten * m_sourceFormat.sampleRate() / m_codecContext->sample_rate);
++ m_recordingEngine.newTimeStamp(time / 1000);
++
++ if constexpr (audioEncoderExtendedTracing)
++ qCDebug(qLcFFmpegAudioEncoder) << "sendPendingFrameToAVCodec" << m_avFrame->nb_samples
++ << m_codecContext->frame_size << m_avFrame->pts;
++
++ int ret = avcodec_send_frame(m_codecContext.get(), m_avFrame.get());
++ if (ret < 0)
++ qCDebug(qLcFFmpegAudioEncoder) << "error sending frame" << ret << QFFmpeg::AVError(ret);
++
++ m_avFrame = nullptr;
++ m_avFrameSamplesOffset = 0;
++ std::fill(m_avFramePlanesData.begin(), m_avFramePlanesData.end(), nullptr);
++ }
++
++ void AudioEncoder::handleAudioData(const uchar *data, int &samplesOffset, int samplesCount)
++ {
++ ensurePendingFrame(samplesCount - samplesOffset);
++
++ writeDataToPendingFrame(data, samplesOffset, samplesCount);
++
++ // The frame is not ready yet
++ if (m_avFrameSamplesOffset < m_avFrame->nb_samples)
++ return;
++
++ retrievePackets();
++
++ sendPendingFrameToAVCodec();
++ }
++
++ } // namespace QFFmpeg
++
++ QT_END_NAMESPACE