Revision control

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegrecordingengine_p.h"
#include "qffmpegencodinginitializer_p.h"
#include "qffmpegaudioencoder_p.h"
#include "qffmpegaudioinput_p.h"
#include "qffmpegrecordingengineutils_p.h"
#include "private/qmultimediautils_p.h"
#include "private/qplatformaudiobufferinput_p.h"
#include "private/qplatformvideosource_p.h"
#include "private/qplatformvideoframeinput_p.h"
#include "qdebug.h"
#include "qffmpegvideoencoder_p.h"
#include "qffmpegmediametadata_p.h"
#include "qffmpegmuxer_p.h"
#include "qloggingcategory.h"
QT_BEGIN_NAMESPACE
namespace ranges = QtMultimediaPrivate::ranges;
Q_STATIC_LOGGING_CATEGORY(qLcFFmpegEncoder, "qt.multimedia.ffmpeg.encoder");
namespace QFFmpeg
{
RecordingEngine::RecordingEngine(const QMediaEncoderSettings &settings,
std::unique_ptr<EncodingFormatContext> context)
: m_settings(settings), m_formatContext(std::move(context)), m_muxer(new Muxer(this))
{
Q_ASSERT(m_formatContext);
Q_ASSERT(m_formatContext->isAVIOOpen());
+ m_startEncodingPromise.start();
}
RecordingEngine::~RecordingEngine()
{
Q_ASSERT(m_state == State::Finalizing);
}
void RecordingEngine::addAudioInput(QFFmpegAudioInput *input)
{
Q_ASSERT(input);
Q_ASSERT(m_state == State::FormatsInitializing);
if (input->device.isNull()) {
emit streamInitializationError(QMediaRecorder::ResourceError,
QLatin1StringView("Audio device is null"));
return;
}
const QAudioFormat format = input->device.preferredFormat();
if (!format.isValid()) {
emit streamInitializationError(
QMediaRecorder::FormatError,
QLatin1StringView("Audio device has invalid preferred format"));
return;
}
AudioEncoder *audioEncoder = createAudioEncoder(format);
connectEncoderToSource(audioEncoder, input);
}
void RecordingEngine::addAudioBufferInput(QPlatformAudioBufferInput *input,
const QAudioBuffer &firstBuffer)
{
Q_ASSERT(input);
Q_ASSERT(m_state == State::FormatsInitializing);
const QAudioFormat format = firstBuffer.isValid() ? firstBuffer.format() : input->audioFormat();
AudioEncoder *audioEncoder = createAudioEncoder(format);
// set the buffer before connecting to avoid potential races
if (firstBuffer.isValid())
audioEncoder->addBuffer(firstBuffer);
connectEncoderToSource(audioEncoder, input);
}
AudioEncoder *RecordingEngine::createAudioEncoder(const QAudioFormat &format)
{
Q_ASSERT(format.isValid());
auto audioEncoder = new AudioEncoder(*this, format, m_settings);
m_audioEncoders.emplace_back(audioEncoder);
connect(audioEncoder, &EncoderThread::endOfSourceStream, this,
&RecordingEngine::handleSourceEndOfStream);
- connect(audioEncoder, &EncoderThread::resolved, this, &RecordingEngine::handleEncoderResolved,
- Qt::SingleShotConnection);
if (m_autoStop)
audioEncoder->setAutoStop(true);
return audioEncoder;
}
void RecordingEngine::addVideoSource(QPlatformVideoSource *source, const QVideoFrame &firstFrame)
{
Q_ASSERT(m_state == State::FormatsInitializing);
QVideoFrameFormat frameFormat =
firstFrame.isValid() ? firstFrame.surfaceFormat() : source->frameFormat();
Q_ASSERT(frameFormat.isValid());
std::optional<AVPixelFormat> hwPixelFormat = source->ffmpegHWPixelFormat()
? AVPixelFormat(*source->ffmpegHWPixelFormat())
: std::optional<AVPixelFormat>{};
qCDebug(qLcFFmpegEncoder) << "adding video source" << source->metaObject()->className() << ":"
<< "pixelFormat=" << frameFormat.pixelFormat()
<< "frameSize=" << frameFormat.frameSize()
<< "frameRate=" << frameFormat.streamFrameRate()
<< "ffmpegHWPixelFormat=" << (hwPixelFormat ? *hwPixelFormat : AV_PIX_FMT_NONE);
auto videoEncoder = new VideoEncoder(*this, m_settings, frameFormat, hwPixelFormat);
m_videoEncoders.emplace_back(videoEncoder);
if (m_autoStop)
videoEncoder->setAutoStop(true);
connect(videoEncoder, &EncoderThread::endOfSourceStream, this,
&RecordingEngine::handleSourceEndOfStream);
- connect(videoEncoder, &EncoderThread::resolved, this, &RecordingEngine::handleEncoderResolved,
- Qt::SingleShotConnection);
-
// set the frame before connecting to avoid potential races
if (firstFrame.isValid())
videoEncoder->addFrame(firstFrame);
connectEncoderToSource(videoEncoder, source);
}
bool RecordingEngine::startEncoders()
{
Q_ASSERT(m_state == State::FormatsInitializing);
Q_ASSERT(m_formatsInitializer);
m_formatsInitializer.reset();
if (m_audioEncoders.empty() && m_videoEncoders.empty()) {
emit sessionError(QMediaRecorder::ResourceError,
QLatin1StringView("No valid stream found for encoding"));
return false;
}
m_state = State::EncodersInitializing;
- forEachEncoder([](EncoderThread *encoder) { //
+ std::vector<QFuture<bool>> resolutions;
+ forEachEncoder([&resolutions](EncoderThread *encoder) { //
encoder->start();
+ resolutions.push_back(encoder->resolvedFuture());
});
+ QtFuture::whenAll(resolutions.begin(), resolutions.end())
+ .then(this, [this](const QList<QFuture<bool>> &resolutions) {
+ handleEncodersResolved(resolutions);
+ });
+
return true;
}
bool RecordingEngine::initialize(const std::vector<QAudioBufferSource *> &audioSources,
const std::vector<QPlatformVideoSource *> &videoSources)
{
Q_ASSERT(m_state == State::None);
m_state = State::FormatsInitializing;
m_formatsInitializer = std::make_unique<EncodingInitializer>(*this);
return m_formatsInitializer->start(audioSources, videoSources);
}
RecordingEngine::EncodingFinalizer::EncodingFinalizer(RecordingEngine &recordingEngine)
: m_recordingEngine(recordingEngine)
{
setObjectName(QStringLiteral("EncodingFinalizer"));
Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
connect(this, &QThread::finished, this, &QObject::deleteLater);
}
void RecordingEngine::EncodingFinalizer::run()
{
Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
- // Block until handleEncoderResolved() (or finalize(), if the encoders had
+ // Block until handleEncodersResolved() (or finalize(), if the encoders had
// already all resolved) has finished with the encoders
- m_encodersResolvedSemaphore.acquire();
+ const bool headerWritten = m_recordingEngine.waitForEncodingStart();
m_recordingEngine.stopAndDeleteThreads();
- if (m_recordingEngine.m_headerWritten) {
+ if (headerWritten) {
const int res = av_write_trailer(m_recordingEngine.avFormatContext());
if (res < 0) {
qCWarning(qLcFFmpegEncoder) << "could not write trailer" << res << AVError(res);
emit m_recordingEngine.sessionError(QMediaRecorder::FormatError,
QLatin1String("Cannot write trailer: ")
+ err2str(res));
}
}
// else ffmpeg might crash
// close AVIO before emitting finalizationDone.
m_recordingEngine.m_formatContext->closeAVIO();
qCDebug(qLcFFmpegEncoder) << "Media recording finalized";
emit m_recordingEngine.finalizationDone();
auto recordingEnginePtr = &m_recordingEngine;
delete recordingEnginePtr;
}
void RecordingEngine::finalize()
{
qCDebug(qLcFFmpegEncoder) << "Media recording finalizing";
Q_ASSERT(m_state == State::FormatsInitializing || m_state == State::EncodersInitializing
|| m_state == State::Encoding);
Q_ASSERT((m_state == State::FormatsInitializing) == !!m_formatsInitializer);
+ if (m_state == State::FormatsInitializing)
+ markEncodingStart(false);
+
m_formatsInitializer.reset();
forEachEncoder(&disconnectEncoderFromSource);
m_state = State::Finalizing;
m_finalizer = new EncodingFinalizer(*this);
-
- // Release finalizer semaphore here or after all encoders are resolved
- if (m_resolvedEncodersCount == encodersCount())
- m_finalizer->releaseEncodersResolved();
-
m_finalizer->start();
}
void RecordingEngine::setPaused(bool paused)
{
forEachEncoder(&EncoderThread::setPaused, paused);
}
void RecordingEngine::setAutoStop(bool autoStop)
{
m_autoStop = autoStop;
forEachEncoder(&EncoderThread::setAutoStop, autoStop);
handleSourceEndOfStream();
}
void RecordingEngine::setMetaData(const QMediaMetaData &metaData)
{
m_metaData = metaData;
}
void RecordingEngine::newTimeStamp(qint64 time)
{
QMutexLocker locker(&m_timeMutex);
if (time > m_timeRecorded) {
m_timeRecorded = time;
emit durationChanged(time);
}
}
bool RecordingEngine::isEndOfSourceStreams() const
{
return allOfEncoders(&EncoderThread::isEndOfSourceStream);
}
void RecordingEngine::handleSourceEndOfStream()
{
if (m_autoStop && isEndOfSourceStreams())
emit autoStopped();
}
- void RecordingEngine::handleEncoderResolved(bool succeeded)
+ void RecordingEngine::handleEncodersResolved(const QList<QFuture<bool>> &resolutions)
{
Q_ASSERT(m_state == State::EncodersInitializing || m_state == State::Finalizing);
- if (!succeeded)
- m_anyEncoderFailed = true;
+ bool allEncodersSucceeded = ranges::all_of(resolutions,
+ [](const QFuture<bool> &future) {
+ return future.result();
+ });
- ++m_resolvedEncodersCount;
-
- Q_ASSERT(m_resolvedEncodersCount <= encodersCount());
-
- if (m_resolvedEncodersCount < encodersCount())
- return; // still waiting on other encoders
-
- if (!m_anyEncoderFailed) {
+ if (allEncodersSucceeded) {
Q_ASSERT(allOfEncoders(&EncoderThread::isInitialized));
qCDebug(qLcFFmpegEncoder) << "Encoders initialized; writing header";
avFormatContext()->metadata = QFFmpegMetaData::toAVMetaData(m_metaData);
const int res = avformat_write_header(avFormatContext(), nullptr);
if (res < 0) {
qWarning() << "could not write header, error:" << res << AVError(res);
- m_anyEncoderFailed = true;
+ allEncodersSucceeded = false;
emit sessionError(QMediaRecorder::ResourceError,
QStringLiteral("Cannot start writing the stream"));
} else {
qCDebug(qLcFFmpegEncoder) << "Stream header is successfully written";
- m_headerWritten = true;
// finalize() may already have moved m_state to Finalizing, don't go backwards
if (m_state == State::EncodersInitializing)
m_state = State::Encoding;
m_muxer->start();
}
}
// Unblock encoders in init()
- forEachEncoder(&EncoderThread::startEncoding, !m_anyEncoderFailed);
-
- // Safe to let a waiting EncodingFinalizer proceed now
- if (m_finalizer)
- m_finalizer->releaseEncodersResolved();
+ markEncodingStart(allEncodersSucceeded);
}
void RecordingEngine::stopAndDeleteThreads()
{
m_audioEncoders.clear();
m_videoEncoders.clear();
m_muxer.reset();
}
+ void RecordingEngine::markEncodingStart(bool startOk)
+ {
+ m_startEncodingPromise.addResult(startOk);
+ m_startEncodingPromise.finish();
+ }
+
template <typename F, typename... Args>
void RecordingEngine::forEachEncoder(F &&f, Args &&...args)
{
for (auto &audioEncoder : m_audioEncoders)
std::invoke(f, audioEncoder.get(), args...);
for (auto &videoEncoder : m_videoEncoders)
std::invoke(f, videoEncoder.get(), args...);
}
template <typename F>
bool RecordingEngine::allOfEncoders(F &&f) const
{
auto predicate = [&f](const auto &encoder) { return std::invoke(f, encoder.get()); };
return ranges::all_of(m_audioEncoders, predicate)
&& ranges::all_of(m_videoEncoders, predicate);
}
} // namespace QFFmpeg
QT_END_NAMESPACE
#include "moc_qffmpegrecordingengine_p.cpp"