Revision control
Copy as Markdown
// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegaudioencoder_p.h"
#include "qffmpegrecordingengineutils_p.h"
#include "qffmpegaudioencoderutils_p.h"
#include "qffmpegaudioinput_p.h"
#include "qffmpegencoderoptions_p.h"
#include "qffmpegmuxer_p.h"
#include "qffmpegrecordingengine_p.h"
#include "qffmpegmediaformatinfo_p.h"
#include "qffmpegcodecstorage_p.h"
#include <QtCore/qloggingcategory.h>
QT_BEGIN_NAMESPACE
namespace QFFmpeg {
Q_STATIC_LOGGING_CATEGORY(qLcFFmpegAudioEncoder, "qt.multimedia.ffmpeg.audioencoder");
static constexpr bool audioEncoderExtendedTracing = false;
namespace {
void setupStreamParameters(AVStream *stream, const Codec &codec,
const AVAudioFormat &requestedAudioFormat)
{
const auto channelLayouts = codec.channelLayouts();
#if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
stream->codecpar->ch_layout =
adjustChannelLayout(channelLayouts, requestedAudioFormat.channelLayout);
#else
stream->codecpar->channel_layout =
adjustChannelLayout(channelLayouts, requestedAudioFormat.channelLayoutMask);
stream->codecpar->channels = qPopulationCount(stream->codecpar->channel_layout);
#endif
const auto sampleRates = codec.sampleRates();
const auto sampleRate = adjustSampleRate(sampleRates, requestedAudioFormat.sampleRate);
stream->codecpar->sample_rate = sampleRate;
// Codecs with a variable frame size (e.g. PCM) must keep frame_size at 0;
// FFmpeg 9 rejects any frame whose nb_samples exceeds a non-zero frame_size,
// even for such codecs. Codecs with a fixed frame size determine and set
// their own frame_size during avcodec_open2, overriding this default.
if (!(codec.capabilities() & AV_CODEC_CAP_VARIABLE_FRAME_SIZE))
stream->codecpar->frame_size = 1024;
const auto sampleFormats = codec.sampleFormats();
stream->codecpar->format = adjustSampleFormat(sampleFormats, requestedAudioFormat.sampleFormat);
stream->time_base = AVRational{ 1, sampleRate };
qCDebug(qLcFFmpegAudioEncoder)
<< "set stream time_base" << stream->time_base.num << "/" << stream->time_base.den;
}
bool openCodecContext(AVCodecContext *codecContext, AVStream *stream,
const QMediaEncoderSettings &settings)
{
Q_ASSERT(codecContext);
codecContext->time_base = stream->time_base;
int status = avcodec_parameters_to_context(codecContext, stream->codecpar);
if (status < 0) {
qCWarning(qLcFFmpegAudioEncoder)
<< "Cannot set codec parameters; result:" << AVError(status);
return false;
}
// if avcodec_open2 fails, it may clean codecContext->codec
Codec codec{ codecContext->codec };
AVDictionaryHolder opts;
applyAudioEncoderOptions(settings, codec.name(), codecContext, opts);
applyExperimentalCodecOptions(codec, opts);
status = avcodec_open2(codecContext, codec.get(), opts);
if (status != 0) {
qCWarning(qLcFFmpegAudioEncoder)
<< "Cannot open audio codec" << codec.name() << "; result:" << AVError(status);
return false;
}
qCDebug(qLcFFmpegAudioEncoder) << "audio codec params: fmt=" << codecContext->sample_fmt
<< "rate=" << codecContext->sample_rate;
status = avcodec_parameters_from_context(stream->codecpar, codecContext);
if (status < 0) {
qCWarning(qLcFFmpegAudioEncoder) << "Cannot get parameters from audio codec context"
<< codec.name() << "; result:" << AVError(status);
return false;
}
return true;
}
} // namespace
AudioEncoder::AudioEncoder(RecordingEngine &recordingEngine, const QAudioFormat &sourceFormat,
const QMediaEncoderSettings &settings)
: EncoderThread(recordingEngine), m_sourceFormat(sourceFormat), m_settings(settings)
{
setObjectName(QLatin1String("AudioEncoder"));
qCDebug(qLcFFmpegAudioEncoder) << "AudioEncoder" << settings.audioCodec();
const AVCodecID codecID = QFFmpegMediaFormatInfo::codecIdForAudioCodec(settings.audioCodec());
Q_ASSERT(avformat_query_codec(recordingEngine.avFormatContext()->oformat, codecID,
FF_COMPLIANCE_NORMAL));
Q_ASSERT(QFFmpeg::findAVEncoder(codecID));
m_stream = avformat_new_stream(recordingEngine.avFormatContext(), nullptr);
m_stream->id = recordingEngine.avFormatContext()->nb_streams - 1;
m_stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
m_stream->codecpar->codec_id = codecID;
}
void AudioEncoder::addBuffer(const QAudioBuffer &buffer)
{
if (!buffer.isValid()) {
setEndOfSourceStream();
return;
}
{
const std::chrono::microseconds bufferDuration(buffer.duration());
auto guard = lockLoopData();
resetEndOfSourceStream();
if (m_paused)
return;
// TODO: apply logic with canPushFrame
m_audioBufferQueue.push(buffer);
m_queueDuration += bufferDuration;
}
dataReady();
}
QAudioBuffer AudioEncoder::takeBuffer()
{
auto locker = lockLoopData();
QAudioBuffer result = dequeueIfPossible(m_audioBufferQueue);
m_queueDuration -= std::chrono::microseconds(result.duration());
return result;
}
bool AudioEncoder::init()
{
const AVAudioFormat requestedAudioFormat(m_sourceFormat);
const AVCodecID codecId = m_stream->codecpar->codec_id;
namespace ranges = QtMultimediaPrivate::ranges;
const auto encoders = findAndScoreEncoders(codecId, [&](const Codec &codec) {
AVScore result = DefaultAVScore;
// Attempt to find no-conversion format
if (auto fmts = codec.sampleFormats(); !fmts.empty())
result += ranges::contains(fmts, requestedAudioFormat.sampleFormat) ? 1 : -1;
if (auto rates = codec.sampleRates(); !rates.empty())
result += ranges::contains(rates, requestedAudioFormat.sampleRate) ? 1 : -1;
#if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
if (auto layouts = codec.channelLayouts(); !layouts.empty())
result += ranges::contains(layouts, requestedAudioFormat.channelLayout) ? 1 : -1;
#else
if (auto layouts = codec.channelLayouts(); !layouts.empty())
result +=
ranges::contains(layouts, requestedAudioFormat.channelLayoutMask) ? 1 : -1;
#endif
return result;
});
for (const auto &[codec, score] : encoders) {
AVCodecContextUPtr codecContext(avcodec_alloc_context3(codec.get()));
if (!codecContext)
continue;
setupStreamParameters(m_stream, Codec{ codecContext->codec }, requestedAudioFormat);
if (!openCodecContext(codecContext.get(), m_stream, m_settings))
continue;
m_codecContext = std::move(codecContext);
break;
}
if (!m_codecContext) {
qCWarning(qLcFFmpegAudioEncoder) << "Unable to open any audio codec";
markResolved(false);
Q_EMIT m_recordingEngine.sessionError(QMediaRecorder::FormatError,
QStringLiteral("Cannot open any audio codec"));
return false;
}
qCDebug(qLcFFmpegAudioEncoder) << "found audio codec" << m_codecContext->codec->name;
updateResampler(m_sourceFormat);
// TODO: try to address this dependency here.
if (auto input = qobject_cast<QFFmpegAudioInput *>(source()))
input->setBufferSize(m_codecContext->frame_size);
return EncoderThread::init();
}
void AudioEncoder::cleanup()
{
while (!m_audioBufferQueue.empty())
processOne();
if (m_avFrameSamplesOffset) {
// the size of the last frame can be less than m_codecContext->frame_size
retrievePackets();
sendPendingFrameToAVCodec();
}
while (avcodec_send_frame(m_codecContext.get(), nullptr) == AVERROR(EAGAIN))
retrievePackets();
retrievePackets();
}
bool AudioEncoder::hasData() const
{
return !m_audioBufferQueue.empty();
}
void AudioEncoder::retrievePackets()
{
while (true) {
AVPacketUPtr packet(av_packet_alloc());
int ret = avcodec_receive_packet(m_codecContext.get(), packet.get());
switch (ret) {
case 0:
break;
case AVERROR(EAGAIN):
case AVERROR(EOF):
return;
default:
qCDebug(qLcFFmpegAudioEncoder) << "receive packet" << ret << QFFmpeg::AVError{ ret };
return;
}
if constexpr (audioEncoderExtendedTracing)
qCDebug(qLcFFmpegAudioEncoder)
<< "writing audio packet" << packet->size << packet->pts << packet->dts;
packet->stream_index = m_stream->id;
m_recordingEngine.getMuxer()->addPacket(std::move(packet));
}
}
void AudioEncoder::processOne()
{
QAudioBuffer buffer = takeBuffer();
Q_ASSERT(buffer.isValid());
if constexpr (audioEncoderExtendedTracing)
qCDebug(qLcFFmpegAudioEncoder)
<< "new audio buffer" << buffer.byteCount() << buffer.format()
<< buffer.frameCount() << m_codecContext->frame_size;
if (buffer.format() != m_sourceFormat && !updateResampler(buffer.format()))
return;
int samplesOffset = 0;
const int bufferSamplesCount = static_cast<int>(buffer.frameCount());
while (samplesOffset < bufferSamplesCount)
handleAudioData(buffer.constData<uint8_t>(), samplesOffset, bufferSamplesCount);
Q_ASSERT(samplesOffset == bufferSamplesCount);
}
bool AudioEncoder::checkIfCanPushFrame() const
{
if (m_encodingStarted)
return m_audioBufferQueue.size() <= 1 || m_queueDuration < m_maxQueueDuration;
if (!isFinished())
return m_audioBufferQueue.empty();
return false;
}
bool AudioEncoder::updateResampler(const QAudioFormat &sourceFormat)
{
m_resampler.reset();
const AVAudioFormat requestedAudioFormat(sourceFormat);
const AVAudioFormat codecAudioFormat(m_codecContext.get());
if (requestedAudioFormat != codecAudioFormat) {
m_resampler = createResampleContext(requestedAudioFormat, codecAudioFormat);
if (!swr_is_initialized(m_resampler.get())) {
m_sourceFormat = {};
qCWarning(qLcFFmpegAudioEncoder) << "Cannot initialize resampler for audio encoder";
Q_EMIT m_recordingEngine.sessionError(
QMediaRecorder::FormatError,
QStringLiteral("Cannot initialize resampler for audio encoder"));
return false;
}
qCDebug(qLcFFmpegAudioEncoder) << "Created resampler with audio formats conversion\n"
<< requestedAudioFormat << "->" << codecAudioFormat;
} else {
qCDebug(qLcFFmpegAudioEncoder) << "Resampler is not needed due to no-conversion format\n"
<< requestedAudioFormat;
}
m_sourceFormat = sourceFormat;
return true;
}
void AudioEncoder::ensurePendingFrame(int availableSamplesCount)
{
Q_ASSERT(availableSamplesCount >= 0);
if (m_avFrame)
return;
m_avFrame = makeAVFrame();
m_avFrame->format = m_codecContext->sample_fmt;
#if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
m_avFrame->ch_layout = m_codecContext->ch_layout;
#else
m_avFrame->channel_layout = m_codecContext->channel_layout;
m_avFrame->channels = m_codecContext->channels;
#endif
m_avFrame->sample_rate = m_codecContext->sample_rate;
const bool isFixedFrameSize =
!(m_codecContext->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)
&& m_codecContext->frame_size;
m_avFrame->nb_samples = isFixedFrameSize ? m_codecContext->frame_size : availableSamplesCount;
if (m_avFrame->nb_samples) {
const int status = av_frame_get_buffer(m_avFrame.get(), 0);
if (status < 0)
qCWarning(qLcFFmpegAudioEncoder)
<< "Failed to allocate audio frame buffer:" << AVError(status);
}
const auto &timeBase = m_stream->time_base;
const auto pts = timeBase.den && timeBase.num
? timeBase.den * m_samplesWritten / (m_codecContext->sample_rate * timeBase.num)
: m_samplesWritten;
setAVFrameTime(*m_avFrame, pts, timeBase);
}
void AudioEncoder::writeDataToPendingFrame(const uchar *data, int &samplesOffset, int samplesCount)
{
Q_ASSERT(m_avFrame);
Q_ASSERT(m_avFrameSamplesOffset <= m_avFrame->nb_samples);
const int bytesPerSample = av_get_bytes_per_sample(m_codecContext->sample_fmt);
const bool isPlanar = av_sample_fmt_is_planar(m_codecContext->sample_fmt);
#if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
const int channelsCount = m_codecContext->ch_layout.nb_channels;
#else
const int channelsCount = m_codecContext->channels;
#endif
const int audioDataOffset = isPlanar ? bytesPerSample * m_avFrameSamplesOffset
: bytesPerSample * m_avFrameSamplesOffset * channelsCount;
const int planesCount = isPlanar ? channelsCount : 1;
m_avFramePlanesData.resize(planesCount);
for (int plane = 0; plane < planesCount; ++plane)
m_avFramePlanesData[plane] = m_avFrame->extended_data[plane] + audioDataOffset;
const int samplesToWrite = m_avFrame->nb_samples - m_avFrameSamplesOffset;
int samplesToRead =
(samplesToWrite * m_sourceFormat.sampleRate() + m_codecContext->sample_rate / 2)
/ m_codecContext->sample_rate;
// the lower bound is need to get round infinite loops in corner cases
samplesToRead = qBound(1, samplesToRead, samplesCount - samplesOffset);
data += m_sourceFormat.bytesForFrames(samplesOffset);
if (m_resampler) {
m_avFrameSamplesOffset += swr_convert(m_resampler.get(), m_avFramePlanesData.data(),
samplesToWrite, &data, samplesToRead);
} else {
Q_ASSERT(planesCount == 1);
m_avFrameSamplesOffset += samplesToRead;
memcpy(m_avFramePlanesData[0], data, m_sourceFormat.bytesForFrames(samplesToRead));
}
samplesOffset += samplesToRead;
}
void AudioEncoder::sendPendingFrameToAVCodec()
{
Q_ASSERT(m_avFrame);
Q_ASSERT(m_avFrameSamplesOffset <= m_avFrame->nb_samples);
m_avFrame->nb_samples = m_avFrameSamplesOffset;
m_samplesWritten += m_avFrameSamplesOffset;
const qint64 time = m_sourceFormat.durationForFrames(
m_samplesWritten * m_sourceFormat.sampleRate() / m_codecContext->sample_rate);
m_recordingEngine.newTimeStamp(time / 1000);
if constexpr (audioEncoderExtendedTracing)
qCDebug(qLcFFmpegAudioEncoder) << "sendPendingFrameToAVCodec" << m_avFrame->nb_samples
<< m_codecContext->frame_size << m_avFrame->pts;
int ret = avcodec_send_frame(m_codecContext.get(), m_avFrame.get());
if (ret < 0)
qCDebug(qLcFFmpegAudioEncoder) << "error sending frame" << ret << QFFmpeg::AVError(ret);
m_avFrame = nullptr;
m_avFrameSamplesOffset = 0;
std::fill(m_avFramePlanesData.begin(), m_avFramePlanesData.end(), nullptr);
}
void AudioEncoder::handleAudioData(const uchar *data, int &samplesOffset, int samplesCount)
{
ensurePendingFrame(samplesCount - samplesOffset);
writeDataToPendingFrame(data, samplesOffset, samplesCount);
// The frame is not ready yet
if (m_avFrameSamplesOffset < m_avFrame->nb_samples)
return;
retrievePackets();
sendPendingFrameToAVCodec();
}
} // namespace QFFmpeg
QT_END_NAMESPACE