Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegrecordingengine_p.h"
#include "qffmpegencodinginitializer_p.h"
#include "qffmpegaudioencoder_p.h"
#include "qffmpegaudioinput_p.h"
#include "qffmpegrecordingengineutils_p.h"
#include "private/qmultimediautils_p.h"
#include "private/qplatformaudiobufferinput_p.h"
#include "private/qplatformvideosource_p.h"
#include "private/qplatformvideoframeinput_p.h"
#include "qdebug.h"
#include "qffmpegvideoencoder_p.h"
#include "qffmpegmediametadata_p.h"
#include "qffmpegmuxer_p.h"
#include "qloggingcategory.h"
extern "C" {
#include "libavutil/opt.h"
}
QT_BEGIN_NAMESPACE
namespace ranges = QtMultimediaPrivate::ranges;
Q_STATIC_LOGGING_CATEGORY(qLcFFmpegEncoder, "qt.multimedia.ffmpeg.encoder");
namespace QFFmpeg
{
namespace {
bool supportsNegativeCtsOffsets(const AVOutputFormat *format)
{
if (!format || !format->priv_class)
return false;
// Check if muxer can store negative composition offsets in a version 1
// ctts box, exposed as the negative_cts_offsets movflag
const AVClass *avClass = format->priv_class;
return av_opt_find(static_cast<void *>(&avClass), "negative_cts_offsets", "movflags", 0, 0)
!= nullptr;
}
AVDictionaryHolder muxerOptions(const AVFormatContext *formatContext)
{
AVDictionaryHolder options;
if (supportsNegativeCtsOffsets(formatContext->oformat)) {
// Codecs that reorder frames (H264, H265) emit packets with dts starting negative (frame
// reorder delay). Muxer can store that delay as negative composition offsets
av_dict_set(options, "movflags", "+negative_cts_offsets", 0);
// Disable avformat shifting to keep packets unmodified for muxer
av_dict_set(options, "avoid_negative_ts", "disabled", 0);
}
return options;
}
} // namespace
RecordingEngine::RecordingEngine(const QMediaEncoderSettings &settings,
std::unique_ptr<EncodingFormatContext> context)
: m_settings(settings), m_formatContext(std::move(context)), m_muxer(new Muxer(this))
{
Q_ASSERT(m_formatContext);
Q_ASSERT(m_formatContext->isAVIOOpen());
m_startEncodingPromise.start();
}
RecordingEngine::~RecordingEngine()
{
Q_ASSERT(m_state == State::Finalizing);
}
void RecordingEngine::addAudioInput(QFFmpegAudioInput *input)
{
Q_ASSERT(input);
Q_ASSERT(m_state == State::FormatsInitializing);
if (input->device.isNull()) {
Q_EMIT streamInitializationError(QMediaRecorder::ResourceError,
QLatin1StringView("Audio device is null"));
return;
}
const QAudioFormat format = input->device.preferredFormat();
if (!format.isValid()) {
Q_EMIT streamInitializationError(
QMediaRecorder::FormatError,
QLatin1StringView("Audio device has invalid preferred format"));
return;
}
AudioEncoder *audioEncoder = createAudioEncoder(format);
connectEncoderToSource(audioEncoder, input);
}
void RecordingEngine::addAudioBufferInput(QPlatformAudioBufferInput *input,
const QAudioBuffer &firstBuffer)
{
Q_ASSERT(input);
Q_ASSERT(m_state == State::FormatsInitializing);
const QAudioFormat format = firstBuffer.isValid() ? firstBuffer.format() : input->audioFormat();
AudioEncoder *audioEncoder = createAudioEncoder(format);
// set the buffer before connecting to avoid potential races
if (firstBuffer.isValid())
audioEncoder->addBuffer(firstBuffer);
connectEncoderToSource(audioEncoder, input);
}
AudioEncoder *RecordingEngine::createAudioEncoder(const QAudioFormat &format)
{
Q_ASSERT(format.isValid());
auto audioEncoder = new AudioEncoder(*this, format, m_settings);
m_audioEncoders.emplace_back(audioEncoder);
connect(audioEncoder, &EncoderThread::endOfSourceStream, this,
&RecordingEngine::handleSourceEndOfStream);
if (m_autoStop)
audioEncoder->setAutoStop(true);
return audioEncoder;
}
void RecordingEngine::addVideoSource(QPlatformVideoSource *source, const QVideoFrame &firstFrame)
{
Q_ASSERT(m_state == State::FormatsInitializing);
QVideoFrameFormat frameFormat =
firstFrame.isValid() ? firstFrame.surfaceFormat() : source->frameFormat();
Q_ASSERT(frameFormat.isValid());
std::optional<AVPixelFormat> hwPixelFormat = source->ffmpegHWPixelFormat()
? AVPixelFormat(*source->ffmpegHWPixelFormat())
: std::optional<AVPixelFormat>{};
qCDebug(qLcFFmpegEncoder) << "adding video source" << source->metaObject()->className() << ":"
<< "pixelFormat=" << frameFormat.pixelFormat()
<< "frameSize=" << frameFormat.frameSize()
<< "frameRate=" << frameFormat.streamFrameRate()
<< "ffmpegHWPixelFormat=" << (hwPixelFormat ? *hwPixelFormat : AV_PIX_FMT_NONE);
auto videoEncoder = new VideoEncoder(*this, m_settings, frameFormat, hwPixelFormat);
m_videoEncoders.emplace_back(videoEncoder);
if (m_autoStop)
videoEncoder->setAutoStop(true);
connect(videoEncoder, &EncoderThread::endOfSourceStream, this,
&RecordingEngine::handleSourceEndOfStream);
// set the frame before connecting to avoid potential races
if (firstFrame.isValid())
videoEncoder->addFrame(firstFrame);
connectEncoderToSource(videoEncoder, source);
}
bool RecordingEngine::startEncoders()
{
Q_ASSERT(m_state == State::FormatsInitializing);
Q_ASSERT(m_formatsInitializer);
m_formatsInitializer.reset();
if (m_audioEncoders.empty() && m_videoEncoders.empty()) {
Q_EMIT sessionError(QMediaRecorder::ResourceError,
QLatin1StringView("No valid stream found for encoding"));
return false;
}
m_state = State::EncodersInitializing;
std::vector<QFuture<bool>> resolutions;
forEachEncoder([&resolutions](EncoderThread *encoder) { //
encoder->start();
resolutions.push_back(encoder->resolvedFuture());
});
QtFuture::whenAll(resolutions.begin(), resolutions.end())
.then(this, [this](const QList<QFuture<bool>> &resolutions) {
handleEncodersResolved(resolutions);
});
return true;
}
bool RecordingEngine::initialize(const std::vector<QAudioBufferSource *> &audioSources,
const std::vector<QPlatformVideoSource *> &videoSources)
{
Q_ASSERT(m_state == State::None);
m_state = State::FormatsInitializing;
m_formatsInitializer = std::make_unique<EncodingInitializer>(*this);
return m_formatsInitializer->start(audioSources, videoSources);
}
RecordingEngine::EncodingFinalizer::EncodingFinalizer(RecordingEngine &recordingEngine)
: m_recordingEngine(recordingEngine)
{
setObjectName(QStringLiteral("EncodingFinalizer"));
Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
connect(this, &QThread::finished, this, &QObject::deleteLater);
}
void RecordingEngine::EncodingFinalizer::run()
{
Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
// Block until handleEncodersResolved() (or finalize(), if the encoders had
// already all resolved) has finished with the encoders
const bool headerWritten = m_recordingEngine.waitForEncodingStart();
m_recordingEngine.stopAndDeleteThreads();
if (headerWritten) {
const int res = av_write_trailer(m_recordingEngine.avFormatContext());
if (res < 0) {
qCWarning(qLcFFmpegEncoder) << "could not write trailer" << res << AVError(res);
Q_EMIT m_recordingEngine.sessionError(QMediaRecorder::FormatError,
QLatin1String("Cannot write trailer: ")
+ err2str(res));
}
}
// else ffmpeg might crash
// close AVIO before emitting finalizationDone.
m_recordingEngine.m_formatContext->closeAVIO();
qCDebug(qLcFFmpegEncoder) << "Media recording finalized";
Q_EMIT m_recordingEngine.finalizationDone();
m_recordingEngine.deleteLater(); // defer destruction to m_recordingEngine's thread
}
void RecordingEngine::finalize()
{
qCDebug(qLcFFmpegEncoder) << "Media recording finalizing";
Q_ASSERT(m_state == State::FormatsInitializing || m_state == State::EncodersInitializing
|| m_state == State::Encoding);
Q_ASSERT((m_state == State::FormatsInitializing) == !!m_formatsInitializer);
if (m_state == State::FormatsInitializing)
markEncodingStart(false);
m_formatsInitializer.reset();
forEachEncoder(&disconnectEncoderFromSource);
m_state = State::Finalizing;
m_finalizer = new EncodingFinalizer(*this);
m_finalizer->start();
}
void RecordingEngine::setPaused(bool paused)
{
forEachEncoder(&EncoderThread::setPaused, paused);
}
void RecordingEngine::setAutoStop(bool autoStop)
{
m_autoStop = autoStop;
forEachEncoder(&EncoderThread::setAutoStop, autoStop);
handleSourceEndOfStream();
}
void RecordingEngine::setMetaData(const QMediaMetaData &metaData)
{
m_metaData = metaData;
}
void RecordingEngine::newTimeStamp(qint64 time)
{
QMutexLocker locker(&m_timeMutex);
if (time > m_timeRecorded) {
m_timeRecorded = time;
Q_EMIT durationChanged(time);
}
}
bool RecordingEngine::isEndOfSourceStreams() const
{
return allOfEncoders(&EncoderThread::isEndOfSourceStream);
}
void RecordingEngine::handleSourceEndOfStream()
{
// Once finalizing, the EncodingFinalizer thread owns the encoders and deletes
// them, so we must not access them. The recording is stopping anyway.
// This can be reached via queued end-of-stream signals emitted before finalize().
if (m_state == State::Finalizing)
return;
if (m_autoStop && isEndOfSourceStreams())
Q_EMIT autoStopped();
}
void RecordingEngine::handleEncodersResolved(const QList<QFuture<bool>> &resolutions)
{
Q_ASSERT(m_state == State::EncodersInitializing || m_state == State::Finalizing);
bool allEncodersSucceeded = ranges::all_of(resolutions,
[](const QFuture<bool> &future) {
return future.result();
});
if (allEncodersSucceeded) {
Q_ASSERT(allOfEncoders(&EncoderThread::isInitialized));
qCDebug(qLcFFmpegEncoder) << "Encoders initialized; writing header";
avFormatContext()->metadata = QFFmpegMetaData::toAVMetaData(m_metaData);
AVDictionaryHolder options = muxerOptions(avFormatContext());
const int res = avformat_write_header(avFormatContext(), options);
if (res < 0) {
qWarning() << "could not write header, error:" << res << AVError(res);
allEncodersSucceeded = false;
Q_EMIT sessionError(QMediaRecorder::ResourceError,
QStringLiteral("Cannot start writing the stream"));
} else {
qCDebug(qLcFFmpegEncoder) << "Stream header is successfully written";
// finalize() may already have moved m_state to Finalizing, don't go backwards
if (m_state == State::EncodersInitializing)
m_state = State::Encoding;
m_muxer->start();
}
}
// Unblock encoders in init()
markEncodingStart(allEncodersSucceeded);
}
void RecordingEngine::stopAndDeleteThreads()
{
m_audioEncoders.clear();
m_videoEncoders.clear();
m_muxer.reset();
}
void RecordingEngine::markEncodingStart(bool startOk)
{
m_startEncodingPromise.addResult(startOk);
m_startEncodingPromise.finish();
}
template <typename F, typename... Args>
void RecordingEngine::forEachEncoder(F &&f, Args &&...args)
{
for (auto &audioEncoder : m_audioEncoders)
std::invoke(f, audioEncoder.get(), args...);
for (auto &videoEncoder : m_videoEncoders)
std::invoke(f, videoEncoder.get(), args...);
}
template <typename F>
bool RecordingEngine::allOfEncoders(F &&f) const
{
auto predicate = [&f](const auto &encoder) { return std::invoke(f, encoder.get()); };
return ranges::all_of(m_audioEncoders, predicate)
&& ranges::all_of(m_videoEncoders, predicate);
}
} // namespace QFFmpeg
QT_END_NAMESPACE
#include "moc_qffmpegrecordingengine_p.cpp"