Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegmediacapturesession_p.h"
#include <QtCore/qloggingcategory.h>
#include <QtFFmpegMediaPluginImpl/private/qffmpegaudioinput_p.h>
#include <QtFFmpegMediaPluginImpl/private/qffmpegimagecapture_p.h>
#include <QtFFmpegMediaPluginImpl/private/qffmpegmediarecorder_p.h>
#include <QtMultimedia/private/qmultimedia_ranges_p.h>
#include <QtMultimedia/private/qplatformaudioinput_p.h>
#include <QtMultimedia/private/qplatformaudiobufferinput_p.h>
#include <QtMultimedia/private/qplatformaudiooutput_p.h>
#include <QtMultimedia/private/qplatformcamera_p.h>
#include <QtMultimedia/private/qplatformsurfacecapture_p.h>
#include <QtMultimedia/private/qplatformvideoframeinput_p.h>
#include <QtMultimedia/qaudiobuffer.h>
#include <QtMultimedia/qaudiooutput.h>
#include <QtMultimedia/qaudiosink.h>
#include <QtMultimedia/qvideosink.h>
#include <array>
QT_BEGIN_NAMESPACE
using namespace Qt::StringLiterals;
namespace ranges = QtMultimediaPrivate::ranges;
Q_STATIC_LOGGING_CATEGORY(qLcFFmpegMediaCaptureSession, "qt.multimedia.ffmpeg.mediacapturesession")
static int preferredAudioSinkBufferSize(const QFFmpegAudioInput &input)
{
// Heuristic params to avoid jittering
// TODO: investigate the reason of jittering and probably reduce the factor
constexpr int BufferSizeFactor = 2;
constexpr int BufferSizeExceeding = 4096;
return input.bufferSize() * BufferSizeFactor + BufferSizeExceeding;
}
QFFmpegMediaCaptureSession::QFFmpegMediaCaptureSession()
{
connect(
this,
&QFFmpegMediaCaptureSession::primaryActiveVideoSourceChanged,
this,
[this] {
updateVideoFrameConnection(nullptr);
});
}
QFFmpegMediaCaptureSession::~QFFmpegMediaCaptureSession() = default;
QPlatformCamera *QFFmpegMediaCaptureSession::camera()
{
return m_camera;
}
void QFFmpegMediaCaptureSession::setCamera(QPlatformCamera *camera)
{
if (setVideoSource(m_camera, camera))
Q_EMIT cameraChanged();
}
QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::screenCapture()
{
return m_screenCapture;
}
void QFFmpegMediaCaptureSession::setScreenCapture(QPlatformSurfaceCapture *screenCapture)
{
if (setVideoSource(m_screenCapture, screenCapture))
Q_EMIT screenCaptureChanged();
}
QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::windowCapture()
{
return m_windowCapture;
}
void QFFmpegMediaCaptureSession::setWindowCapture(QPlatformSurfaceCapture *windowCapture)
{
if (setVideoSource(m_windowCapture, windowCapture))
Q_EMIT windowCaptureChanged();
}
QPlatformVideoFrameInput *QFFmpegMediaCaptureSession::videoFrameInput()
{
return m_videoFrameInput;
}
void QFFmpegMediaCaptureSession::setVideoFrameInput(QPlatformVideoFrameInput *input)
{
if (setVideoSource(m_videoFrameInput, input))
Q_EMIT videoFrameInputChanged();
}
QPlatformImageCapture *QFFmpegMediaCaptureSession::imageCapture()
{
return m_imageCapture;
}
void QFFmpegMediaCaptureSession::setImageCapture(QPlatformImageCapture *imageCapture)
{
if (m_imageCapture == imageCapture)
return;
if (m_imageCapture)
m_imageCapture->setCaptureSession(nullptr);
m_imageCapture = static_cast<QFFmpegImageCapture *>(imageCapture);
if (m_imageCapture)
m_imageCapture->setCaptureSession(this);
Q_EMIT imageCaptureChanged();
}
void QFFmpegMediaCaptureSession::setMediaRecorder(QPlatformMediaRecorder *recorder)
{
auto *r = static_cast<QFFmpegMediaRecorder *>(recorder);
if (m_mediaRecorder == r)
return;
if (m_mediaRecorder)
m_mediaRecorder->setCaptureSession(nullptr);
m_mediaRecorder = r;
if (m_mediaRecorder)
m_mediaRecorder->setCaptureSession(this);
Q_EMIT encoderChanged();
}
QPlatformMediaRecorder *QFFmpegMediaCaptureSession::mediaRecorder()
{
return m_mediaRecorder;
}
void QFFmpegMediaCaptureSession::setAudioInput(QPlatformAudioInput *input)
{
qCDebug(qLcFFmpegMediaCaptureSession)
<< "set audio input:" << (input ? input->device.description() : u"null"_s);
auto ffmpegAudioInput = dynamic_cast<QFFmpegAudioInput *>(input);
Q_ASSERT(!!input == !!ffmpegAudioInput);
if (m_audioInput == ffmpegAudioInput)
return;
if (m_audioInput)
m_audioInput->q->disconnect(this);
m_audioInput = ffmpegAudioInput;
if (m_audioInput)
// TODO: implement the signal in QPlatformAudioInput and connect to it, QTBUG-112294
connect(m_audioInput->q, &QAudioInput::deviceChanged, this,
&QFFmpegMediaCaptureSession::updateAudioSink);
updateAudioSink();
}
void QFFmpegMediaCaptureSession::setAudioBufferInput(QPlatformAudioBufferInput *input)
{
// TODO: implement binding to audio sink like setAudioInput does
m_audioBufferInput = input;
}
void QFFmpegMediaCaptureSession::updateAudioSink()
{
if (m_audioSink) {
m_audioSink->reset();
m_audioSink.reset();
}
if (!m_audioInput || !m_audioOutput)
return;
auto format = m_audioInput->device.preferredFormat();
if (!m_audioOutput->device.isFormatSupported(format))
qWarning() << "Audio source format" << format << "is not compatible with the audio output";
m_audioSink = std::make_unique<QAudioSink>(m_audioOutput->device, format);
m_audioBufferSize = preferredAudioSinkBufferSize(*m_audioInput);
m_audioSink->setBufferSize(m_audioBufferSize);
qCDebug(qLcFFmpegMediaCaptureSession)
<< "Create audiosink, format:" << format << "bufferSize:" << m_audioSink->bufferSize()
<< "output device:" << m_audioOutput->device.description();
m_audioIODevice = m_audioSink->start();
if (m_audioIODevice) {
auto writeToDevice = [this](const QAudioBuffer &buffer) {
if (m_audioBufferSize < preferredAudioSinkBufferSize(*m_audioInput)) {
qCDebug(qLcFFmpegMediaCaptureSession)
<< "Recreate audiosink due to small buffer size:" << m_audioBufferSize;
updateAudioSink();
}
const auto written =
m_audioIODevice->write(buffer.data<const char>(), buffer.byteCount());
if (written < buffer.byteCount())
qCWarning(qLcFFmpegMediaCaptureSession)
<< "Not all bytes written:" << written << "vs" << buffer.byteCount();
};
connect(m_audioInput, &QFFmpegAudioInput::newAudioBuffer, m_audioSink.get(), writeToDevice);
} else {
qWarning() << "Failed to start audiosink push mode";
}
updateVolume();
}
void QFFmpegMediaCaptureSession::updateVolume()
{
if (m_audioSink)
m_audioSink->setVolume(m_audioOutput->muted ? 0.f : m_audioOutput->volume);
}
QPlatformAudioInput *QFFmpegMediaCaptureSession::audioInput() const
{
return m_audioInput;
}
void QFFmpegMediaCaptureSession::setVideoPreview(QVideoSink *sink)
{
if (std::exchange(m_videoSink, sink) == sink)
return;
updateVideoFrameConnection(nullptr);
}
void QFFmpegMediaCaptureSession::setAudioOutput(QPlatformAudioOutput *output)
{
qCDebug(qLcFFmpegMediaCaptureSession)
<< "set audio output:" << (output ? output->device.description() : u"null"_s);
if (m_audioOutput == output)
return;
if (m_audioOutput)
m_audioOutput->q->disconnect(this);
m_audioOutput = output;
if (m_audioOutput) {
// TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294
connect(m_audioOutput->q, &QAudioOutput::deviceChanged, this,
&QFFmpegMediaCaptureSession::updateAudioSink);
connect(m_audioOutput->q, &QAudioOutput::volumeChanged, this,
&QFFmpegMediaCaptureSession::updateVolume);
connect(m_audioOutput->q, &QAudioOutput::mutedChanged, this,
&QFFmpegMediaCaptureSession::updateVolume);
}
updateAudioSink();
}
void QFFmpegMediaCaptureSession::onSourceActivating(QPlatformVideoSource &source)
{
// The source is about to go active and may emit frames from a
// background thread before setActive() is done executing.
// We set up the connections ahead of time to make sure
// always catch the first frame(s).
updateVideoFrameConnection(&source);
}
void QFFmpegMediaCaptureSession::onSourceActivationFailure(QPlatformVideoSource &)
{
// The source failed to activate, so it will not become active. Revert to the
// connection implied by the currently active sources.
updateVideoFrameConnection(nullptr);
}
// Returns the highest-priority active video source. If pendingActiveSource
// is set, it is treated as active even if it does not report isActive() yet.
QPlatformVideoSource *
QFFmpegMediaCaptureSession::primaryVideoSource(QPlatformVideoSource *pendingActiveSource)
{
// Priority order must match QPlatformMediaCaptureSession::activeVideoSources().
const std::array<QPlatformVideoSource *, 4> sourcesByPriority{
videoFrameInput(), camera(), screenCapture(), windowCapture()
};
auto it = ranges::find_if(
sourcesByPriority,
[&](QPlatformVideoSource *item) {
return item && (item == pendingActiveSource || item->isActive());
});
if (it != sourcesByPriority.end())
return *it;
return nullptr;
}
void QFFmpegMediaCaptureSession::updateVideoFrameConnection(QPlatformVideoSource *pendingActiveSource)
{
QPlatformVideoSource *source = primaryVideoSource(pendingActiveSource);
if (m_videoSinkConnection.source == source && m_videoSinkConnection.sink == m_videoSink)
return;
disconnect(m_videoSinkConnection.connection);
m_videoSinkConnection = {};
if (source && m_videoSink) {
// AutoConnection type might be a pessimization due to an extra queuing
// TODO: investigate and integrate direct connection
m_videoSinkConnection.source = source;
m_videoSinkConnection.sink = m_videoSink;
m_videoSinkConnection.connection = connect(
source,
&QPlatformVideoSource::newVideoFrame,
m_videoSink,
&QVideoSink::setVideoFrame);
}
}
void QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource()
{
QPlatformVideoSource *source = primaryVideoSource(nullptr);
if (std::exchange(m_primaryActiveVideoSource, source) != source)
Q_EMIT primaryActiveVideoSourceChanged();
}
template<typename VideoSource>
bool QFFmpegMediaCaptureSession::setVideoSource(QPointer<VideoSource> &source,
VideoSource *newSource)
{
if (source == newSource)
return false;
if (auto prevSource = std::exchange(source, newSource)) {
prevSource->setCaptureSession(nullptr);
prevSource->disconnect(this);
}
if (source) {
source->setCaptureSession(this);
connect(source, &QPlatformVideoSource::activeChanged, this,
&QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource);
connect(source, &QObject::destroyed, this,
&QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource, Qt::QueuedConnection);
}
updatePrimaryActiveVideoSource();
return true;
}
QPlatformVideoSource *QFFmpegMediaCaptureSession::primaryActiveVideoSource()
{
return m_primaryActiveVideoSource;
}
std::vector<QAudioBufferSource *> QFFmpegMediaCaptureSession::activeAudioInputs() const
{
std::vector<QAudioBufferSource *> result;
if (m_audioInput)
result.push_back(m_audioInput);
if (m_audioBufferInput)
result.push_back(m_audioBufferInput);
return result;
}
QT_END_NAMESPACE
#include "moc_qffmpegmediacapturesession_p.cpp"