Revision control
++ // Copyright (C) 2016 Jolla Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ #include "qgstvideorenderersink_p.h"
++
++ #include <QtMultimedia/qvideoframe.h>
++ #include <QtMultimedia/qvideosink.h>
++ #include <QtMultimedia/private/qvideoframe_p.h>
++ #include <QtGui/rhi/qrhi.h>
++ #include <QtGui/qguiapplication.h>
++ #include <QtGui/qopenglcontext.h>
++ #include <QtCore/qcoreapplication.h>
++ #include <QtCore/qdebug.h>
++ #include <QtCore/qloggingcategory.h>
++ #include <QtCore/private/qfactoryloader_p.h>
++ #include <QtCore/private/quniquehandle_p.h>
++
++ #include <common/qgst_debug_p.h>
++ #include <common/qgstreamermetadata_p.h>
++ #include <common/qgstreamervideosink_p.h>
++ #include <common/qgstutils_p.h>
++ #include <common/qgstvideobuffer_p.h>
++
++ #include <gst/video/video.h>
++ #include <gst/video/gstvideometa.h>
++
++
++ #if QT_CONFIG(gstreamer_gl)
++ #include <gst/gl/gl.h>
++ #endif // #if QT_CONFIG(gstreamer_gl)
++
++ // DMA support
++ #if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
++ # include <common/qgstreameregldisplay_p.h>
++ # include <gst/allocators/gstdmabuf.h>
++ #endif
++
++ // NOLINTBEGIN(readability-convert-member-functions-to-static)
++
++ Q_STATIC_LOGGING_CATEGORY(qLcGstVideoRenderer, "qt.multimedia.gstvideorenderer");
++
++ QT_BEGIN_NAMESPACE
++
++ QGstVideoRenderer::QGstVideoRenderer(QGstreamerRelayVideoSink *sink)
++ : m_sink(sink), m_surfaceCaps(createSurfaceCaps(sink))
++ {
++ QObject::connect(
++ sink, &QGstreamerRelayVideoSink::aboutToBeDestroyed, this,
++ [this] {
++ QMutexLocker locker(&m_sinkMutex);
++ m_sink = nullptr;
++ },
++ Qt::DirectConnection);
++ }
++
++ QGstVideoRenderer::~QGstVideoRenderer() = default;
++
++ QGstCaps QGstVideoRenderer::createSurfaceCaps([[maybe_unused]] QGstreamerRelayVideoSink *sink)
++ {
++ QGstCaps caps = QGstCaps::create();
++
++ // All the formats that both we and gstreamer support
++ auto formats = QList<QVideoFrameFormat::PixelFormat>()
++ << QVideoFrameFormat::Format_YUV420P
++ << QVideoFrameFormat::Format_YUV422P
++ << QVideoFrameFormat::Format_YV12
++ << QVideoFrameFormat::Format_UYVY
++ << QVideoFrameFormat::Format_YUYV
++ << QVideoFrameFormat::Format_NV12
++ << QVideoFrameFormat::Format_NV21
++ << QVideoFrameFormat::Format_AYUV
++ << QVideoFrameFormat::Format_P010
++ << QVideoFrameFormat::Format_XRGB8888
++ << QVideoFrameFormat::Format_XBGR8888
++ << QVideoFrameFormat::Format_RGBX8888
++ << QVideoFrameFormat::Format_BGRX8888
++ << QVideoFrameFormat::Format_ARGB8888
++ << QVideoFrameFormat::Format_ABGR8888
++ << QVideoFrameFormat::Format_RGBA8888
++ << QVideoFrameFormat::Format_BGRA8888
++ << QVideoFrameFormat::Format_Y8
++ << QVideoFrameFormat::Format_Y16
++ ;
++ caps.addPixelFormats(formats);
++ #if QT_CONFIG(gstreamer_gl)
++ QRhi *rhi = sink->rhi();
++ if (rhi && rhi->backend() == QRhi::OpenGLES2) {
++ caps.addPixelFormats(formats, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
++ # if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
++ if (qGstEglCanMapDmaBuf(rhi)) {
++ caps.addPixelFormats(formats, GST_CAPS_FEATURE_MEMORY_DMABUF);
++ }
++ # endif
++ }
++ #endif
++ return caps;
++ }
++
++ void QGstVideoRenderer::customEvent(QEvent *event)
++ {
++ QT_WARNING_PUSH
++ QT_WARNING_DISABLE_GCC("-Wswitch") // case value not in enumerated type ‘QEvent::Type’
++
++ switch (event->type()) {
++ case renderFramesEvent: {
++ // LATER: we currently show every frame. however it may be reasonable to drop frames
++ // here if the queue contains more than one frame
++ while (std::optional<RenderBufferState> nextState = m_bufferQueue.dequeue())
++ handleNewBuffer(std::move(*nextState));
++ return;
++ }
++ case stopEvent: {
++ m_currentPipelineFrame = {};
++ updateCurrentVideoFrame(m_currentVideoFrame);
++ return;
++ }
++
++ default:
++ return;
++ }
++ QT_WARNING_POP
++ }
++
++
++ void QGstVideoRenderer::handleNewBuffer(RenderBufferState state)
++ {
++ QVideoFrame frame =
++ qCreateFrameFromGstBuffer(state.buffer, state.videoInfo, state.format);
++ QGstUtils::setFrameTimeStampsFromBuffer(&frame, state.buffer.get());
++ m_currentPipelineFrame = std::move(frame);
++
++ if (!m_isActive) {
++ qCDebug(qLcGstVideoRenderer) << " showing empty video frame";
++ updateCurrentVideoFrame({});
++ return;
++ }
++
++ updateCurrentVideoFrame(m_currentPipelineFrame);
++ }
++
++ const QGstCaps &QGstVideoRenderer::caps()
++ {
++ return m_surfaceCaps;
++ }
++
++ bool QGstVideoRenderer::start(const QGstCaps& caps)
++ {
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::start" << caps;
++
++ auto optionalVideoInfo = caps.videoInfo();
++ if (optionalVideoInfo) {
++ m_videoInfo = std::move(*optionalVideoInfo);
++ m_format = qVideoFrameFormatFromGstVideoInfo(m_videoInfo);
++ } else {
++ m_format = {};
++ m_videoInfo = {};
++ }
++ m_capsMemoryFormat = caps.memoryFormat();
++
++ // NOTE: m_format will not be fully populated until GST_EVENT_TAG is processed
++
++ return true;
++ }
++
++ void QGstVideoRenderer::stop()
++ {
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::stop";
++
++ m_bufferQueue.clear();
++ QCoreApplication::postEvent(this, new QEvent(stopEvent));
++ }
++
++ void QGstVideoRenderer::unlock()
++ {
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::unlock";
++ }
++
++ bool QGstVideoRenderer::proposeAllocation(GstQuery *query)
++ {
++ if (!query || GST_QUERY_TYPE(query) != GST_QUERY_ALLOCATION)
++ return false;
++
++ GstCaps *queryCaps = nullptr;
++ gboolean needPool = false;
++ gst_query_parse_allocation(query, &queryCaps, &needPool);
++ GstVideoInfo info;
++ int size = 0;
++ if (queryCaps && gst_video_info_from_caps(&info, queryCaps)) {
++ size = info.size;
++ } else {
++ qWarning(qLcGstVideoRenderer) << "QGstVideoRenderer::proposeAllocation failed to "
++ "get size from query caps";
++ return true;
++ }
++
++ constexpr int defaultMinBuffers = 3;
++ static const int env
++ = qEnvironmentVariableIntValue("QT_GSTREAMER_PROPOSE_ALLOCATION_MIN_BUFFERS");
++ static const int minBuffers = env ? env : defaultMinBuffers;
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::proposeAllocation: "
++ "needPool:" << needPool
++ << "size:" << size
++ << "minBuffers:" << minBuffers;
++
++ // This call is needed to avoid enabling of copy threshold by v4l2 decoders, which can result
++ // in a mix of dmabuf and system memory buffers. The query sender should use its own buffer
++ // pool, and will only take the size value and our suggested minimum buffers into account.
++ // The driver can force a higher minimum if minBuffers is set too low, making 3 sufficient.
++ gst_query_add_allocation_pool(query, nullptr, size, minBuffers, 0);
++
++ // Advertise GstVideoMeta support, often needed for DMA buffer negotiation.
++ gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, nullptr);
++
++ return true;
++ }
++
++ GstFlowReturn QGstVideoRenderer::render(GstBuffer *buffer)
++ {
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::render";
++
++ if (m_flushing) {
++ qCDebug(qLcGstVideoRenderer)
++ << " buffer received while flushing the sink ... discarding buffer";
++ return GST_FLOW_FLUSHING;
++ }
++
++ GstVideoCropMeta *meta = gst_buffer_get_video_crop_meta(buffer);
++ if (meta) {
++ QRect vp(meta->x, meta->y, meta->width, meta->height);
++ if (m_format.viewport() != vp) {
++ qCDebug(qLcGstVideoRenderer)
++ << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x"
++ << meta->width << " | " << meta->x << "x" << meta->y << "]";
++ // Update viewport if data is not the same
++ m_format.setViewport(vp);
++ }
++ }
++
++ // Some gst elements, like v4l2h264dec, can provide Direct Memory Access buffers (DMA-BUF)
++ // without specifying it in their caps. So we check the memory format manually:
++ QGstCaps::MemoryFormat bufferMemoryFormat = [&] {
++ if (m_capsMemoryFormat != QGstCaps::CpuMemory)
++ return m_capsMemoryFormat;
++
++ return qMemoryFormatFromGstBuffer(buffer);
++ }();
++
++ qCDebug(qLcGstVideoRenderer) << "m_capsMemoryFormat" << m_capsMemoryFormat
++ << "bufferMemoryFormat" << bufferMemoryFormat;
++
++ QVideoFrameFormat bufferVideoFrameFormat = m_format;
++
++ // When rendering DMAbuf frames with OpenGLES, EGL seems to do implicit YUV->RGB conversion for
++ // UYVY and YUYV (YUY2), so we need to change the pixel format to Format_RGBA8888 to select the
++ // correct shader.
++ const bool setFormat_RGBA8888 = [&] {
++ #if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
++ QRhi *rhi = m_sink ? m_sink->rhi() : nullptr;
++ if ((m_format.pixelFormat() == QVideoFrameFormat::Format_UYVY
++ || m_format.pixelFormat() == QVideoFrameFormat::Format_YUYV)
++ && bufferMemoryFormat == QGstCaps::DMABuf && qGstEglCanMapDmaBuf(rhi)) {
++
++ if (!rhi || rhi->backend() != QRhi::OpenGLES2)
++ return false;
++
++ auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
++ QOpenGLContext *glContext = nativeHandles ? nativeHandles->context : nullptr;
++ return glContext && glContext->isOpenGLES();
++ }
++ #endif
++ return false;
++ }();
++ if (setFormat_RGBA8888) {
++ // TODO: Replace with new private setter of pixel format.
++ qCDebug(qLcGstVideoRenderer) << "Setting pixel format to Format_RGBA8888";
++ bufferVideoFrameFormat = QVideoFrameFormat(m_format.frameSize(),
++ QVideoFrameFormat::Format_RGBA8888);
++ bufferVideoFrameFormat.setStreamFrameRate(m_format.streamFrameRate());
++ bufferVideoFrameFormat.setColorRange(m_format.colorRange());
++ bufferVideoFrameFormat.setColorTransfer(m_format.colorTransfer());
++ bufferVideoFrameFormat.setColorSpace(m_format.colorSpace());
++ }
++
++ RenderBufferState state{
++ QGstBufferHandle{ buffer, QGstBufferHandle::NeedsRef },
++ bufferVideoFrameFormat,
++ m_videoInfo,
++ bufferMemoryFormat,
++ };
++
++ qCDebug(qLcGstVideoRenderer) << " sending video frame";
++
++ qsizetype sizeOfQueue = m_bufferQueue.enqueue(std::move(state));
++ if (sizeOfQueue == 1)
++ // we only need to wake up, if we don't have a pending frame
++ QCoreApplication::postEvent(this, new QEvent(renderFramesEvent));
++
++ return GST_FLOW_OK;
++ }
++
++ bool QGstVideoRenderer::query(GstQuery *query)
++ {
++ #if QT_CONFIG(gstreamer_gl)
++ if (GST_QUERY_TYPE(query) == GST_QUERY_CONTEXT) {
++ const gchar *type = nullptr;
++ gst_query_parse_context_type(query, &type);
++
++ QLatin1StringView typeStr(type);
++ if (typeStr != QLatin1StringView("gst.gl.GLDisplay")
++ && typeStr != QLatin1StringView("gst.gl.local_context")) {
++ return false;
++ }
++
++ QMutexLocker locker(&m_sinkMutex);
++ if (!m_sink)
++ return false;
++
++ auto *gstGlContext = typeStr == QLatin1StringView("gst.gl.GLDisplay")
++ ? m_sink->gstGlDisplayContext() : m_sink->gstGlLocalContext();
++ if (!gstGlContext)
++ return false;
++
++ gst_query_set_context(query, gstGlContext);
++
++ return true;
++ }
++ #else
++ Q_UNUSED(query);
++ #endif
++ return false;
++ }
++
++ void QGstVideoRenderer::gstEvent(GstEvent *event)
++ {
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent:" << event;
++
++ switch (GST_EVENT_TYPE(event)) {
++ case GST_EVENT_TAG:
++ return gstEventHandleTag(event);
++ case GST_EVENT_EOS:
++ return gstEventHandleEOS(event);
++ case GST_EVENT_FLUSH_START:
++ return gstEventHandleFlushStart(event);
++ case GST_EVENT_FLUSH_STOP:
++ return gstEventHandleFlushStop(event);
++
++ default:
++ return;
++ }
++ }
++
++ void QGstVideoRenderer::setActive(bool isActive)
++ {
++ if (isActive == m_isActive)
++ return;
++
++ m_isActive = isActive;
++ if (isActive)
++ updateCurrentVideoFrame(m_currentPipelineFrame);
++ else
++ updateCurrentVideoFrame({});
++ }
++
++ void QGstVideoRenderer::updateCurrentVideoFrame(QVideoFrame frame)
++ {
++ m_currentVideoFrame = std::move(frame);
++ if (m_sink)
++ m_sink->setVideoFrame(m_currentVideoFrame);
++ }
++
++ void QGstVideoRenderer::gstEventHandleTag(GstEvent *event)
++ {
++ GstTagList *taglist = nullptr;
++ gst_event_parse_tag(event, &taglist);
++ if (!taglist)
++ return;
++
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEventHandleTag:" << taglist;
++
++ QGString value;
++ if (!gst_tag_list_get_string(taglist, GST_TAG_IMAGE_ORIENTATION, &value))
++ return;
++
++ RotationResult parsed = parseRotationTag(value.get());
++
++ m_format.setMirrored(parsed.flip);
++ m_format.setRotation(parsed.rotation);
++ }
++
++ void QGstVideoRenderer::gstEventHandleEOS(GstEvent *)
++ {
++ stop();
++ }
++
++ void QGstVideoRenderer::gstEventHandleFlushStart(GstEvent *)
++ {
++ // "data is to be discarded"
++ m_flushing = true;
++ m_bufferQueue.clear();
++ }
++
++ void QGstVideoRenderer::gstEventHandleFlushStop(GstEvent *)
++ {
++ // "data is allowed again"
++ m_flushing = false;
++ }
++
++ static GstVideoSinkClass *gvrs_sink_parent_class;
++ static thread_local QGstreamerRelayVideoSink *gvrs_current_sink;
++
++ #define VO_SINK(s) QGstVideoRendererSink *sink(reinterpret_cast<QGstVideoRendererSink *>(s))
++
++ QGstVideoRendererSinkElement QGstVideoRendererSink::createSink(QGstreamerRelayVideoSink *sink)
++ {
++ setSink(sink);
++ QGstVideoRendererSink *gstSink = reinterpret_cast<QGstVideoRendererSink *>(
++ g_object_new(QGstVideoRendererSink::get_type(), nullptr));
++
++ return QGstVideoRendererSinkElement{
++ gstSink,
++ QGstElement::NeedsRef,
++ };
++ }
++
++ void QGstVideoRendererSink::setSink(QGstreamerRelayVideoSink *sink)
++ {
++ gvrs_current_sink = sink;
++ }
++
++ GType QGstVideoRendererSink::get_type()
++ {
++ static const GTypeInfo info =
++ {
++ sizeof(QGstVideoRendererSinkClass), // class_size
++ base_init, // base_init
++ nullptr, // base_finalize
++ class_init, // class_init
++ nullptr, // class_finalize
++ nullptr, // class_data
++ sizeof(QGstVideoRendererSink), // instance_size
++ 0, // n_preallocs
++ instance_init, // instance_init
++ nullptr // value_table
++ };
++
++ static const GType type = g_type_register_static(GST_TYPE_VIDEO_SINK, "QGstVideoRendererSink",
++ &info, GTypeFlags(0));
++
++ return type;
++ }
++
++ void QGstVideoRendererSink::class_init(gpointer g_class, gpointer class_data)
++ {
++ Q_UNUSED(class_data);
++
++ gvrs_sink_parent_class = reinterpret_cast<GstVideoSinkClass *>(g_type_class_peek_parent(g_class));
++
++ GstVideoSinkClass *video_sink_class = reinterpret_cast<GstVideoSinkClass *>(g_class);
++ video_sink_class->show_frame = QGstVideoRendererSink::show_frame;
++
++ GstBaseSinkClass *base_sink_class = reinterpret_cast<GstBaseSinkClass *>(g_class);
++ base_sink_class->get_caps = QGstVideoRendererSink::get_caps;
++ base_sink_class->set_caps = QGstVideoRendererSink::set_caps;
++ base_sink_class->propose_allocation = QGstVideoRendererSink::propose_allocation;
++ base_sink_class->stop = QGstVideoRendererSink::stop;
++ base_sink_class->unlock = QGstVideoRendererSink::unlock;
++ base_sink_class->query = QGstVideoRendererSink::query;
++ base_sink_class->event = QGstVideoRendererSink::event;
++
++ GstElementClass *element_class = reinterpret_cast<GstElementClass *>(g_class);
++ element_class->change_state = QGstVideoRendererSink::change_state;
++ gst_element_class_set_metadata(element_class,
++ "Qt built-in video renderer sink",
++ "Sink/Video",
++ "Qt default built-in video renderer sink",
++ "The Qt Company");
++
++ GObjectClass *object_class = reinterpret_cast<GObjectClass *>(g_class);
++ object_class->finalize = QGstVideoRendererSink::finalize;
++ }
++
++ void QGstVideoRendererSink::base_init(gpointer g_class)
++ {
++ static GstStaticPadTemplate sink_pad_template = GST_STATIC_PAD_TEMPLATE(
++ "sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(
++ "video/x-raw, "
++ "framerate = (fraction) [ 0, MAX ], "
++ "width = (int) [ 1, MAX ], "
++ "height = (int) [ 1, MAX ]"));
++
++ gst_element_class_add_pad_template(
++ GST_ELEMENT_CLASS(g_class), gst_static_pad_template_get(&sink_pad_template));
++ }
++
++ void QGstVideoRendererSink::instance_init(GTypeInstance *instance, gpointer g_class)
++ {
++ Q_UNUSED(g_class);
++ VO_SINK(instance);
++
++ Q_ASSERT(gvrs_current_sink);
++
++ sink->renderer = new QGstVideoRenderer(gvrs_current_sink);
++ sink->renderer->moveToThread(gvrs_current_sink->thread());
++ gvrs_current_sink = nullptr;
++ }
++
++ void QGstVideoRendererSink::finalize(GObject *object)
++ {
++ VO_SINK(object);
++
++ delete sink->renderer;
++
++ // Chain up
++ G_OBJECT_CLASS(gvrs_sink_parent_class)->finalize(object);
++ }
++
++ GstStateChangeReturn QGstVideoRendererSink::change_state(
++ GstElement *element, GstStateChange transition)
++ {
++ GstStateChangeReturn ret =
++ GST_ELEMENT_CLASS(gvrs_sink_parent_class)->change_state(element, transition);
++ qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::change_state:" << transition << ret;
++ return ret;
++ }
++
++ GstCaps *QGstVideoRendererSink::get_caps(GstBaseSink *base, GstCaps *filter)
++ {
++ VO_SINK(base);
++
++ QGstCaps caps = sink->renderer->caps();
++ if (filter)
++ caps = QGstCaps(gst_caps_intersect(caps.caps(), filter), QGstCaps::HasRef);
++
++ return caps.release();
++ }
++
++ gboolean QGstVideoRendererSink::set_caps(GstBaseSink *base, GstCaps *gcaps)
++ {
++ VO_SINK(base);
++ auto caps = QGstCaps(gcaps, QGstCaps::NeedsRef);
++
++ qCDebug(qLcGstVideoRenderer) << "set_caps:" << caps;
++
++ if (!caps) {
++ sink->renderer->stop();
++ return TRUE;
++ }
++
++ return sink->renderer->start(caps);
++ }
++
++ gboolean QGstVideoRendererSink::propose_allocation(GstBaseSink *base, GstQuery *query)
++ {
++ VO_SINK(base);
++ return sink->renderer->proposeAllocation(query);
++ }
++
++ gboolean QGstVideoRendererSink::stop(GstBaseSink *base)
++ {
++ VO_SINK(base);
++ sink->renderer->stop();
++ return TRUE;
++ }
++
++ gboolean QGstVideoRendererSink::unlock(GstBaseSink *base)
++ {
++ VO_SINK(base);
++ sink->renderer->unlock();
++ return TRUE;
++ }
++
++ GstFlowReturn QGstVideoRendererSink::show_frame(GstVideoSink *base, GstBuffer *buffer)
++ {
++ VO_SINK(base);
++ return sink->renderer->render(buffer);
++ }
++
++ gboolean QGstVideoRendererSink::query(GstBaseSink *base, GstQuery *query)
++ {
++ VO_SINK(base);
++ if (sink->renderer->query(query))
++ return TRUE;
++
++ return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->query(base, query);
++ }
++
++ gboolean QGstVideoRendererSink::event(GstBaseSink *base, GstEvent * event)
++ {
++ VO_SINK(base);
++ sink->renderer->gstEvent(event);
++ return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->event(base, event);
++ }
++
++ QGstVideoRendererSinkElement::QGstVideoRendererSinkElement(QGstVideoRendererSink *element,
++ RefMode mode)
++ : QGstBaseSink{
++ qGstCheckedCast<GstBaseSink>(element),
++ mode,
++ }
++ {
++ }
++
++ void QGstVideoRendererSinkElement::setActive(bool isActive)
++ {
++ qGstVideoRendererSink()->renderer->setActive(isActive);
++ }
++
++ QGstVideoRendererSink *QGstVideoRendererSinkElement::qGstVideoRendererSink() const
++ {
++ return reinterpret_cast<QGstVideoRendererSink *>(element());
++ }
++
++ QT_END_NAMESPACE