Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include <mediacapture/qgstreamermediacapturesession_p.h>
#include <mediacapture/qgstreamermediarecorder_p.h>
#include <mediacapture/qgstreamerimagecapture_p.h>
#include <mediacapture/qgstreamercamera_p.h>
#include <common/qgstpipeline_p.h>
#include <common/qgstreameraudioinput_p.h>
#include <common/qgstreameraudiooutput_p.h>
#include <common/qgstreamervideooutput_p.h>
#include <common/qgst_debug_p.h>
#include <QtMultimedia/private/qthreadlocalrhi_p.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/private/quniquehandle_p.h>
QT_BEGIN_NAMESPACE
namespace {
QGstElement makeTee(const char *name)
{
QGstElement tee = QGstElement::createFromFactory("tee", name);
tee.set("allow-not-linked", true);
return tee;
}
template <typename Functor>
void executeWhilePadsAreIdle(QSpan<QGstPad> pads, Functor &&f)
{
if (pads.isEmpty())
f();
if (!pads.front())
return executeWhilePadsAreIdle(pads.subspan(1), f);
if (pads.size() == 1)
pads.front().modifyPipelineInIdleProbe(f);
else {
auto remain = pads.subspan(1);
pads.front().modifyPipelineInIdleProbe([&] {
executeWhilePadsAreIdle(remain, f);
});
}
}
void setStateOnElements(QSpan<const QGstElement> elements, GstState state)
{
for (QGstElement element : elements)
if (element)
element.setState(state);
}
void finishStateChangeOnElements(QSpan<const QGstElement> elements)
{
for (QGstElement element : elements)
if (element)
element.finishStateChange();
}
} // namespace
q23::expected<QPlatformMediaCaptureSession *, QString> QGstreamerMediaCaptureSession::create()
{
auto videoOutput = QGstreamerVideoOutput::create();
if (!videoOutput)
return q23::unexpected{ videoOutput.error() };
static const auto error = qGstErrorMessageIfElementsNotAvailable("tee", "capsfilter");
if (error)
return q23::unexpected{ *error };
return new QGstreamerMediaCaptureSession(videoOutput.value());
}
QGstreamerMediaCaptureSession::QGstreamerMediaCaptureSession(QGstreamerVideoOutput *videoOutput)
: m_capturePipeline{
QGstPipeline::create("mediaCapturePipeline"),
},
m_gstAudioTee{
makeTee("audioTee"),
},
m_audioSrcPadForEncoder{ m_gstAudioTee.getRequestPad("src_%u") },
m_audioSrcPadForOutput{ m_gstAudioTee.getRequestPad("src_%u") },
m_gstVideoTee{
makeTee("videoTee"),
},
m_videoSrcPadForEncoder{ m_gstVideoTee.getRequestPad("src_%u") },
m_videoSrcPadForOutput{ m_gstVideoTee.getRequestPad("src_%u") },
m_videoSrcPadForImageCapture{ m_gstVideoTee.getRequestPad("src_%u") },
m_gstVideoOutput(videoOutput)
{
m_gstVideoOutput->setParent(this);
// NOTE: Creating a GStreamer video sink to be owned by the capture session, any sink created by
// user would be a pluggable sink connected to this
m_gstVideoSink = new QGstreamerRelayVideoSink(this);
m_gstVideoOutput->setVideoSink(m_gstVideoSink);
m_gstVideoOutput->setIsPreview();
m_capturePipeline.installMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this));
m_capturePipeline.set("message-forward", true);
// Use system clock to drive all elements in the pipeline. Otherwise,
// the clock is sourced from the elements (e.g. from an audio source).
// Since the elements are added and removed dynamically the clock would
// also change causing lost of synchronization in the pipeline.
QGstClockHandle systemClock{
gst_system_clock_obtain(),
QGstClockHandle::HasRef,
};
gst_pipeline_use_clock(m_capturePipeline.pipeline(), systemClock.get());
// This is the recording pipeline with only live sources, thus the pipeline
// will be always in the playing state.
m_capturePipeline.setState(GST_STATE_PLAYING);
m_gstVideoOutput->setActive(true);
m_capturePipeline.dumpGraph("initial");
}
QGstPad QGstreamerMediaCaptureSession::imageCaptureSink()
{
return m_imageCapture ? m_imageCapture->gstElement().staticPad("sink") : QGstPad{};
}
QGstPad QGstreamerMediaCaptureSession::videoOutputSink()
{
return m_gstVideoOutput ? m_gstVideoOutput->gstElement().staticPad("sink") : QGstPad{};
}
QGstPad QGstreamerMediaCaptureSession::audioOutputSink()
{
return m_gstAudioOutput ? m_gstAudioOutput->gstElement().staticPad("sink") : QGstPad{};
}
QGstreamerMediaCaptureSession::~QGstreamerMediaCaptureSession()
{
setMediaRecorder(nullptr);
setImageCapture(nullptr);
setCamera(nullptr);
m_capturePipeline.removeMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this));
m_capturePipeline.setStateSync(GST_STATE_READY);
m_capturePipeline.setStateSync(GST_STATE_NULL);
}
QPlatformCamera *QGstreamerMediaCaptureSession::camera()
{
return m_gstCamera;
}
void QGstreamerMediaCaptureSession::setCamera(QPlatformCamera *platformCamera)
{
auto *camera = static_cast<QGstreamerCameraBase *>(platformCamera);
if (m_gstCamera == camera)
return;
if (m_gstCamera) {
QObject::disconnect(m_gstCameraActiveConnection);
if (m_gstVideoTee)
setCameraActive(false);
}
m_gstCamera = camera;
if (m_gstCamera) {
m_gstCameraActiveConnection =
QObject::connect(camera, &QPlatformCamera::activeChanged, this,
&QGstreamerMediaCaptureSession::setCameraActive);
if (m_gstCamera->isActive())
setCameraActive(true);
}
Q_EMIT cameraChanged();
}
void QGstreamerMediaCaptureSession::setCameraActive(bool activate)
{
std::array padsToSync = {
m_videoSrcPadForEncoder,
m_videoSrcPadForImageCapture,
m_videoSrcPadForOutput,
m_gstVideoTee.sink(),
};
QGstElement cameraElement = m_gstCamera->gstElement();
QGstElement videoOutputElement = m_gstVideoOutput->gstElement();
if (activate) {
m_capturePipeline.add(m_gstVideoTee);
executeWhilePadsAreIdle(padsToSync, [&] {
m_capturePipeline.add(cameraElement);
if (videoOutputElement)
m_capturePipeline.add(videoOutputElement);
if (m_currentRecorderState && m_currentRecorderState->videoSink)
m_videoSrcPadForEncoder.link(m_currentRecorderState->videoSink);
if (videoOutputElement)
m_videoSrcPadForOutput.link(videoOutputSink());
if (m_imageCapture)
m_videoSrcPadForImageCapture.link(imageCaptureSink());
qLinkGstElements(cameraElement, m_gstVideoTee);
setStateOnElements({ m_gstVideoTee, cameraElement, videoOutputElement },
GST_STATE_PLAYING);
});
finishStateChangeOnElements({ m_gstVideoTee, cameraElement, videoOutputElement });
for (QGstElement addedElement : { m_gstVideoTee, cameraElement, videoOutputElement })
addedElement.finishStateChange();
} else {
executeWhilePadsAreIdle(padsToSync, [&] {
for (QGstPad &pad : padsToSync)
pad.unlinkPeer();
});
m_capturePipeline.stopAndRemoveElements(cameraElement, m_gstVideoTee, videoOutputElement);
}
m_capturePipeline.dumpGraph("camera");
}
QPlatformImageCapture *QGstreamerMediaCaptureSession::imageCapture()
{
return m_imageCapture;
}
void QGstreamerMediaCaptureSession::setImageCapture(QPlatformImageCapture *imageCapture)
{
QGstreamerImageCapture *control = static_cast<QGstreamerImageCapture *>(imageCapture);
if (m_imageCapture == control)
return;
m_videoSrcPadForEncoder.modifyPipelineInIdleProbe([&] {
if (m_imageCapture) {
qUnlinkGstElements(m_gstVideoTee, m_imageCapture->gstElement());
m_capturePipeline.stopAndRemoveElements(m_imageCapture->gstElement());
m_imageCapture->setCaptureSession(nullptr);
}
m_imageCapture = control;
if (m_imageCapture) {
m_capturePipeline.add(m_imageCapture->gstElement());
m_videoSrcPadForImageCapture.link(imageCaptureSink());
m_imageCapture->setCaptureSession(this);
m_imageCapture->gstElement().setState(GST_STATE_PLAYING);
}
});
if (m_imageCapture)
m_imageCapture->gstElement().finishStateChange();
m_capturePipeline.dumpGraph("imageCapture");
Q_EMIT imageCaptureChanged();
}
void QGstreamerMediaCaptureSession::setMediaRecorder(QPlatformMediaRecorder *recorder)
{
QGstreamerMediaRecorder *control = static_cast<QGstreamerMediaRecorder *>(recorder);
if (m_mediaRecorder == control)
return;
if (m_mediaRecorder)
m_mediaRecorder->setCaptureSession(nullptr);
m_mediaRecorder = control;
if (m_mediaRecorder)
m_mediaRecorder->setCaptureSession(this);
Q_EMIT encoderChanged();
m_capturePipeline.dumpGraph("encoder");
}
QPlatformMediaRecorder *QGstreamerMediaCaptureSession::mediaRecorder()
{
return m_mediaRecorder;
}
void QGstreamerMediaCaptureSession::linkAndStartEncoder(RecorderElements recorder,
const QMediaMetaData &metadata)
{
Q_ASSERT(!m_currentRecorderState);
std::array padsToSync = {
m_audioSrcPadForEncoder,
m_videoSrcPadForEncoder,
};
executeWhilePadsAreIdle(padsToSync, [&] {
m_capturePipeline.add(recorder.encodeBin, recorder.fileSink);
qLinkGstElements(recorder.encodeBin, recorder.fileSink);
applyMetaDataToTagSetter(metadata, recorder.encodeBin);
if (recorder.videoSink) {
QGstCaps capsFromCamera = m_gstVideoTee.sink().currentCaps();
m_encoderVideoCapsFilter =
QGstElement::createFromFactory("capsfilter", "encoderVideoCapsFilter");
m_encoderVideoCapsFilter.set("caps", capsFromCamera);
m_capturePipeline.add(m_encoderVideoCapsFilter);
m_encoderVideoCapsFilter.src().link(recorder.videoSink);
m_videoSrcPadForEncoder.link(m_encoderVideoCapsFilter.sink());
}
if (recorder.audioSink) {
QGstCaps capsFromInput = m_gstAudioTee.sink().currentCaps();
m_encoderAudioCapsFilter =
QGstElement::createFromFactory("capsfilter", "encoderAudioCapsFilter");
m_encoderAudioCapsFilter.set("caps", capsFromInput);
m_capturePipeline.add(m_encoderAudioCapsFilter);
m_encoderAudioCapsFilter.src().link(recorder.audioSink);
m_audioSrcPadForEncoder.link(m_encoderAudioCapsFilter.sink());
}
setStateOnElements({ recorder.encodeBin, recorder.fileSink, m_encoderVideoCapsFilter,
m_encoderAudioCapsFilter },
GST_STATE_PLAYING);
GstEvent *event = gst_event_new_reconfigure();
gst_element_send_event(recorder.fileSink.element(), event);
});
finishStateChangeOnElements({ recorder.encodeBin, recorder.fileSink, m_encoderVideoCapsFilter,
m_encoderAudioCapsFilter });
m_currentRecorderState = std::move(recorder);
}
void QGstreamerMediaCaptureSession::unlinkRecorder()
{
std::array padsToSync = {
m_audioSrcPadForEncoder,
m_videoSrcPadForEncoder,
};
executeWhilePadsAreIdle(padsToSync, [&] {
if (m_encoderVideoCapsFilter)
qUnlinkGstElements(m_gstVideoTee, m_encoderVideoCapsFilter);
if (m_encoderAudioCapsFilter)
qUnlinkGstElements(m_gstAudioTee, m_encoderAudioCapsFilter);
});
if (m_encoderVideoCapsFilter) {
m_capturePipeline.stopAndRemoveElements(m_encoderVideoCapsFilter);
m_encoderVideoCapsFilter = {};
}
if (m_encoderAudioCapsFilter) {
m_capturePipeline.stopAndRemoveElements(m_encoderAudioCapsFilter);
m_encoderAudioCapsFilter = {};
}
m_currentRecorderState->encodeBin.sendEos();
}
void QGstreamerMediaCaptureSession::finalizeRecorder()
{
m_capturePipeline.stopAndRemoveElements(m_currentRecorderState->encodeBin,
m_currentRecorderState->fileSink);
m_currentRecorderState = std::nullopt;
}
const QGstPipeline &QGstreamerMediaCaptureSession::pipeline() const
{
return m_capturePipeline;
}
void QGstreamerMediaCaptureSession::setAudioInput(QPlatformAudioInput *input)
{
if (m_gstAudioInput == input)
return;
if (input && !m_gstAudioInput) {
// a new input is connected, we need to add/link the audio tee and audio tee
m_capturePipeline.add(m_gstAudioTee);
std::array padsToSync = {
m_audioSrcPadForEncoder,
m_audioSrcPadForOutput,
m_gstAudioTee.sink(),
};
executeWhilePadsAreIdle(padsToSync, [&] {
if (m_currentRecorderState && m_currentRecorderState->audioSink)
m_audioSrcPadForEncoder.link(m_currentRecorderState->audioSink);
if (m_gstAudioOutput) {
m_capturePipeline.add(m_gstAudioOutput->gstElement());
m_audioSrcPadForOutput.link(audioOutputSink());
}
m_gstAudioInput = static_cast<QGstreamerAudioInput *>(input);
m_capturePipeline.add(m_gstAudioInput->gstElement());
qLinkGstElements(m_gstAudioInput->gstElement(), m_gstAudioTee);
m_gstAudioTee.setState(GST_STATE_PLAYING);
if (m_gstAudioOutput)
m_gstAudioOutput->gstElement().setState(GST_STATE_PLAYING);
m_gstAudioInput->gstElement().setState(GST_STATE_PLAYING);
});
} else if (!input && m_gstAudioInput) {
// input has been removed, unlink and remove audio output and audio tee
std::array padsToSync = {
m_audioSrcPadForEncoder,
m_audioSrcPadForOutput,
m_gstAudioTee.sink(),
};
executeWhilePadsAreIdle(padsToSync, [&] {
for (QGstPad &pad : padsToSync)
pad.unlinkPeer();
});
m_capturePipeline.stopAndRemoveElements(m_gstAudioTee);
if (m_gstAudioOutput)
m_capturePipeline.stopAndRemoveElements(m_gstAudioOutput->gstElement());
m_capturePipeline.stopAndRemoveElements(m_gstAudioInput->gstElement());
m_gstAudioInput = nullptr;
} else {
QGstElement oldInputElement = m_gstAudioInput->gstElement();
m_gstAudioTee.sink().modifyPipelineInIdleProbe([&] {
oldInputElement.sink().unlinkPeer();
m_gstAudioInput = static_cast<QGstreamerAudioInput *>(input);
m_capturePipeline.add(m_gstAudioInput->gstElement());
qLinkGstElements(m_gstAudioInput->gstElement(), m_gstAudioTee);
m_gstAudioInput->gstElement().setState(GST_STATE_PLAYING);
});
m_gstAudioInput->gstElement().finishStateChange();
m_capturePipeline.stopAndRemoveElements(m_gstAudioInput->gstElement());
}
}
void QGstreamerMediaCaptureSession::setVideoPreview(QVideoSink *sink)
{
// Disconnect previous sink
m_gstVideoSink->disconnectPluggableVideoSink();
if (!sink)
return;
// Connect pluggable sink to native sink
auto pluggableSink = dynamic_cast<QGstreamerPluggableVideoSink *>(sink->platformVideoSink());
Q_ASSERT(pluggableSink);
m_gstVideoSink->connectPluggableVideoSink(pluggableSink);
}
void QGstreamerMediaCaptureSession::setAudioOutput(QPlatformAudioOutput *output)
{
if (m_gstAudioOutput == output)
return;
auto *gstOutput = static_cast<QGstreamerAudioOutput *>(output);
if (gstOutput)
gstOutput->setAsync(false);
if (!m_gstAudioInput) {
// audio output is not active, since there is no audio input
m_gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output);
} else {
QGstElement oldOutputElement =
m_gstAudioOutput ? m_gstAudioOutput->gstElement() : QGstElement{};
m_gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output);
m_audioSrcPadForOutput.modifyPipelineInIdleProbe([&] {
if (oldOutputElement)
oldOutputElement.sink().unlinkPeer();
if (m_gstAudioOutput) {
m_capturePipeline.add(m_gstAudioOutput->gstElement());
m_audioSrcPadForOutput.link(m_gstAudioOutput->gstElement().staticPad("sink"));
m_gstAudioOutput->gstElement().setState(GST_STATE_PLAYING);
}
});
if (m_gstAudioOutput)
m_gstAudioOutput->gstElement().finishStateChange();
if (oldOutputElement)
m_capturePipeline.stopAndRemoveElements(oldOutputElement);
}
}
QGstreamerRelayVideoSink *QGstreamerMediaCaptureSession::gstreamerVideoSink() const
{
return m_gstVideoOutput ? m_gstVideoOutput->gstreamerVideoSink() : nullptr;
}
bool QGstreamerMediaCaptureSession::processBusMessage(const QGstreamerMessage &msg)
{
if (m_mediaRecorder)
m_mediaRecorder->processBusMessage(msg);
switch (msg.type()) {
case GST_MESSAGE_ERROR:
return processBusMessageError(msg);
case GST_MESSAGE_LATENCY:
return processBusMessageLatency(msg);
default:
break;
}
return false;
}
bool QGstreamerMediaCaptureSession::processBusMessageError(const QGstreamerMessage &msg)
{
QUniqueGErrorHandle error;
QUniqueGStringHandle message;
gst_message_parse_error(msg.message(), &error, &message);
qWarning() << "QGstreamerMediaCapture: received error from gstreamer" << error << message;
m_capturePipeline.dumpGraph("captureError");
return false;
}
bool QGstreamerMediaCaptureSession::processBusMessageLatency(const QGstreamerMessage &)
{
m_capturePipeline.recalculateLatency();
return false;
}
QT_END_NAMESPACE