Source code
Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qsgvideonode_p.h"
#include <QtMultimedia/private/qhwvideobuffer_p.h>
#include <QtMultimedia/private/qvideoframetexturepool_p.h>
#include <QtMultimedia/private/qvideotexturehelper_p.h>
#include <QtMultimediaQuick/private/qsgvideotexture_p.h>
#include <QtMultimediaQuick/private/qquickvideooutput_p.h>
#include <QtQuick/qsgmaterial.h>
#include <QtQuick/private/qquickitem_p.h>
#include <QtQuick/private/qsginternaltextnode_p.h>
#if QT_CONFIG(opengles2)
#include <QtGui/private/qshaderdescription_p.h>
#endif
QT_BEGIN_NAMESPACE
/* Helpers */
static inline void qSetGeom(QSGGeometry::TexturedPoint2D *v, const QPointF &p)
{
v->x = p.x();
v->y = p.y();
}
static inline void qSetTex(QSGGeometry::TexturedPoint2D *v, const QPointF &p)
{
v->tx = p.x();
v->ty = p.y();
}
static inline void qSwapTex(QSGGeometry::TexturedPoint2D *v0, QSGGeometry::TexturedPoint2D *v1)
{
auto tvx = v0->tx;
auto tvy = v0->ty;
v0->tx = v1->tx;
v0->ty = v1->ty;
v1->tx = tvx;
v1->ty = tvy;
}
class QSGVideoMaterial;
class QSGVideoMaterialRhiShader : public QSGMaterialShader
{
public:
QSGVideoMaterialRhiShader(const QVideoFrameFormat &videoFormat,
const QRhiSwapChain::Format surfaceFormat,
const QRhiSwapChainHdrInfo &hdrInfo,
QRhi *rhi)
: m_videoFormat(videoFormat)
, m_surfaceFormat(surfaceFormat)
, m_hdrInfo(hdrInfo)
{
setShaderFileName(VertexStage, QVideoTextureHelper::vertexShaderFileName(m_videoFormat));
if (QVideoTextureHelper::forceGlTextureExternalOesIsSet()
&& rhi && rhi->backend() == QRhi::OpenGLES2)
setupExternalOESShader();
else
setShaderFileName(FragmentStage, QVideoTextureHelper::fragmentShaderFileName(
m_videoFormat, rhi, m_surfaceFormat));
}
void setupExternalOESShader();
bool updateUniformData(RenderState &state, QSGMaterial *newMaterial,
QSGMaterial *oldMaterial) override;
void updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
protected:
QVideoFrameFormat m_videoFormat;
QRhiSwapChain::Format m_surfaceFormat;
QRhiSwapChainHdrInfo m_hdrInfo;
};
void QSGVideoMaterialRhiShader::setupExternalOESShader()
{
#if QT_CONFIG(opengles2)
qDebug() << "QSGVideoMaterialRhiShader: Setting up external OES shader for OpenGLES2";
using namespace Qt::Literals::StringLiterals;
QByteArray fragmentShader = R"(
#extension GL_OES_EGL_image_external : require
precision highp float;
varying vec2 texCoord;
uniform samplerExternalOES tex0;
void main()
{
gl_FragColor = texture2D(tex0, texCoord);
}
)"_ba;
QShaderDescription desc;
QShaderDescriptionPrivate *descData = QShaderDescriptionPrivate::get(&desc);
QShaderDescription::InOutVariable texCoordInput;
texCoordInput.name = "texCoord";
texCoordInput.type = QShaderDescription::Vec2;
texCoordInput.location = 0;
descData->inVars = { texCoordInput };
QShaderDescription::InOutVariable fragColorOutput;
fragColorOutput.name = "gl_FragColor";
fragColorOutput.type = QShaderDescription::Vec4;
fragColorOutput.location = 0;
descData->outVars = { fragColorOutput };
QShaderDescription::InOutVariable samplerTex0;
samplerTex0.name = "tex0";
samplerTex0.type = QShaderDescription::SamplerExternalOES;
samplerTex0.binding = 1;
descData->combinedImageSamplers = { samplerTex0 };
QShader shaderPack;
shaderPack.setStage(QShader::FragmentStage);
shaderPack.setDescription(desc);
shaderPack.setShader(QShaderKey(QShader::GlslShader, QShaderVersion(100, QShaderVersion::GlslEs)),
QShaderCode(fragmentShader));
setShader(FragmentStage, shaderPack);
#endif
}
class QSGVideoMaterial final : public QSGMaterial
{
public:
QSGVideoMaterial(const QVideoFrameFormat &videoFormat, QRhi *rhi);
[[nodiscard]] QSGMaterialType *type() const override {
static constexpr int NFormats = QRhiSwapChain::HDRExtendedDisplayP3Linear + 1;
static QSGMaterialType type[QVideoFrameFormat::NPixelFormats][NFormats];
return &type[m_videoFormat.pixelFormat()][m_surfaceFormat];
}
[[nodiscard]] QSGMaterialShader *createShader(QSGRendererInterface::RenderMode) const override {
return new QSGVideoMaterialRhiShader(m_videoFormat, m_surfaceFormat, m_hdrInfo, m_rhi);
}
int compare(const QSGMaterial *other) const override {
const QSGVideoMaterial *m = static_cast<const QSGVideoMaterial *>(other);
qint64 diff = m_textures[0].comparisonKey() - m->m_textures[0].comparisonKey();
if (!diff)
diff = m_textures[1].comparisonKey() - m->m_textures[1].comparisonKey();
if (!diff)
diff = m_textures[2].comparisonKey() - m->m_textures[2].comparisonKey();
return diff < 0 ? -1 : (diff > 0 ? 1 : 0);
}
void updateBlending() {
// ### respect video formats with Alpha
setFlag(Blending, !QtPrivate::fuzzyCompare(m_opacity, float(1.0)));
}
void setSurfaceFormat(const QRhiSwapChain::Format surfaceFormat)
{
m_surfaceFormat = surfaceFormat;
}
void setHdrInfo(const QRhiSwapChainHdrInfo &hdrInfo)
{
m_hdrInfo = hdrInfo;
}
void updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates);
const QVideoFrameFormat m_videoFormat;
QRhiSwapChain::Format m_surfaceFormat = QRhiSwapChain::SDR;
float m_opacity = 1.0f;
QRhiSwapChainHdrInfo m_hdrInfo{};
QVideoFrameTexturePoolPtr m_texturePool = std::make_shared<QVideoFrameTexturePool>();
std::array<QSGVideoTexture, 3> m_textures;
QRhi *const m_rhi;
};
void QSGVideoMaterial::updateTextures(QRhi *rhi, QRhiResourceUpdateBatch *resourceUpdates)
{
if (!m_texturePool->texturesDirty())
return;
QVideoFrameTextures *textures = m_texturePool->updateTextures(*rhi, *resourceUpdates);
if (!textures)
return;
for (int plane = 0; plane < 3; ++plane)
m_textures[plane].setRhiTexture(textures->texture(plane));
}
bool QSGVideoMaterialRhiShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial,
QSGMaterial *oldMaterial)
{
Q_UNUSED(oldMaterial);
auto m = static_cast<QSGVideoMaterial *>(newMaterial);
if (!state.isMatrixDirty() && !state.isOpacityDirty())
return false;
if (state.isOpacityDirty()) {
m->m_opacity = state.opacity();
m->updateBlending();
}
// Do this here, not in updateSampledImage. First, with multiple textures we want to
// do this once. More importantly, on some platforms (Android) the externalMatrix is
// updated by this function and we need that already in updateUniformData.
m->updateTextures(state.rhi(), state.resourceUpdateBatch());
float maxNits = 100; // Default to de-facto SDR nits
if (m_surfaceFormat == QRhiSwapChain::HDRExtendedSrgbLinear) {
if (m_hdrInfo.limitsType == QRhiSwapChainHdrInfo::ColorComponentValue)
maxNits = 100 * m_hdrInfo.limits.colorComponentValue.maxColorComponentValue;
else
maxNits = m_hdrInfo.limits.luminanceInNits.maxLuminance;
}
QVideoTextureHelper::updateUniformData(state.uniformData(), m->m_rhi, m_videoFormat,
m->m_texturePool->currentFrame(), state.combinedMatrix(),
state.opacity(), maxNits);
return true;
}
void QSGVideoMaterialRhiShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
QSGMaterial *newMaterial, QSGMaterial *oldMaterial)
{
Q_UNUSED(state);
Q_UNUSED(oldMaterial);
if (binding < 1 || binding > 3)
return;
auto m = static_cast<QSGVideoMaterial *>(newMaterial);
*texture = &m->m_textures[binding - 1];
}
QSGVideoMaterial::QSGVideoMaterial(const QVideoFrameFormat &videoFormat, QRhi *rhi)
: m_videoFormat(videoFormat),
m_rhi(rhi)
{
setFlag(Blending, false);
}
QSGVideoNode::QSGVideoNode(QQuickVideoOutput *parent, const QVideoFrameFormat &videoFormat,
QRhi *rhi)
: m_parent(parent),
m_videoFormat(videoFormat),
m_material{
new QSGVideoMaterial(videoFormat, rhi),
}
{
setFlag(QSGNode::OwnsMaterial);
setFlag(QSGNode::OwnsGeometry);
setMaterial(m_material);
}
QSGVideoNode::~QSGVideoNode()
{
delete m_subtitleTextNode;
}
void QSGVideoNode::setCurrentFrame(const QVideoFrame &frame)
{
texturePool()->setCurrentFrame(frame);
markDirty(DirtyMaterial);
updateSubtitle(frame);
}
void QSGVideoNode::setSurfaceFormat(const QRhiSwapChain::Format surfaceFormat)
{
m_material->setSurfaceFormat(surfaceFormat);
markDirty(DirtyMaterial);
}
void QSGVideoNode::setHdrInfo(const QRhiSwapChainHdrInfo &hdrInfo)
{
m_material->setHdrInfo(hdrInfo);
markDirty(DirtyMaterial);
}
void QSGVideoNode::updateSubtitle(const QVideoFrame &frame)
{
QSize subtitleFrameSize = m_rect.size().toSize();
if (subtitleFrameSize.isEmpty())
return;
subtitleFrameSize = qRotatedFrameSize(subtitleFrameSize, m_videoOutputTransformation.rotation);
if (!m_subtitleLayout.update(subtitleFrameSize, frame.subtitleText()))
return;
delete m_subtitleTextNode;
m_subtitleTextNode = nullptr;
if (frame.subtitleText().isEmpty())
return;
QQuickItemPrivate *parent_d = QQuickItemPrivate::get(m_parent);
m_subtitleTextNode = parent_d->sceneGraphContext()->createInternalTextNode(parent_d->sceneGraphRenderContext());
m_subtitleTextNode->setColor(Qt::white);
QColor bgColor = Qt::black;
bgColor.setAlpha(128);
m_subtitleTextNode->addRectangleNode(m_subtitleLayout.bounds, bgColor, nullptr);
m_subtitleTextNode->addTextLayout(m_subtitleLayout.layout.position(),
&m_subtitleLayout.layout, nullptr);
appendChildNode(m_subtitleTextNode);
setSubtitleGeometry();
}
void QSGVideoNode::setSubtitleGeometry()
{
if (!m_subtitleTextNode)
return;
if (m_material)
updateSubtitle(texturePool()->currentFrame());
float rotate = -1.f * qToUnderlying(m_videoOutputTransformation.rotation);
float yTranslate = 0;
float xTranslate = 0;
if (m_videoOutputTransformation.rotation == QtVideo::Rotation::Clockwise90) {
yTranslate = m_rect.height();
} else if (m_videoOutputTransformation.rotation == QtVideo::Rotation::Clockwise180) {
yTranslate = m_rect.height();
xTranslate = m_rect.width();
} else if (m_videoOutputTransformation.rotation == QtVideo::Rotation::Clockwise270) {
xTranslate = m_rect.width();
}
QMatrix4x4 transform;
transform.translate(m_rect.x() + xTranslate, m_rect.y() + yTranslate);
transform.rotate(rotate, 0, 0, 1);
// TODO: Investigate if we should we mirror subtitles
// if (m_videoOutputTransformation.mirroredHorizontallyAfterRotation)
// transform.scale(-1.f, 1.f);
m_subtitleTextNode->setMatrix(transform);
m_subtitleTextNode->markDirty(DirtyGeometry);
}
/* Update the vertices and texture coordinates.*/
void QSGVideoNode::setTexturedRectGeometry(const QRectF &rect, const QRectF &textureRect,
VideoTransformation videoOutputTransformation)
{
const VideoTransformation currentFrameTransformation = qNormalizedFrameTransformation(
m_material ? texturePool()->currentFrame() : QVideoFrame{}, videoOutputTransformation);
if (rect == m_rect && textureRect == m_textureRect
&& videoOutputTransformation == m_videoOutputTransformation
&& currentFrameTransformation == m_frameTransformation)
return;
m_rect = rect;
m_textureRect = textureRect;
m_videoOutputTransformation = videoOutputTransformation;
m_frameTransformation = currentFrameTransformation;
QSGGeometry *g = geometry();
if (g == nullptr)
g = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4);
QSGGeometry::TexturedPoint2D *v = g->vertexDataAsTexturedPoint2D();
// Vertexes:
// 0 2
//
// 1 3
// Set geometry first
qSetGeom(v + 0, rect.topLeft());
qSetGeom(v + 1, rect.bottomLeft());
qSetGeom(v + 2, rect.topRight());
qSetGeom(v + 3, rect.bottomRight());
// and then texture coordinates
switch (currentFrameTransformation.rotation) {
default:
// tl, bl, tr, br
qSetTex(v + 0, textureRect.topLeft());
qSetTex(v + 1, textureRect.bottomLeft());
qSetTex(v + 2, textureRect.topRight());
qSetTex(v + 3, textureRect.bottomRight());
break;
case QtVideo::Rotation::Clockwise90:
// bl, br, tl, tr
qSetTex(v + 0, textureRect.bottomLeft());
qSetTex(v + 1, textureRect.bottomRight());
qSetTex(v + 2, textureRect.topLeft());
qSetTex(v + 3, textureRect.topRight());
break;
case QtVideo::Rotation::Clockwise180:
// br, tr, bl, tl
qSetTex(v + 0, textureRect.bottomRight());
qSetTex(v + 1, textureRect.topRight());
qSetTex(v + 2, textureRect.bottomLeft());
qSetTex(v + 3, textureRect.topLeft());
break;
case QtVideo::Rotation::Clockwise270:
// tr, tl, br, bl
qSetTex(v + 0, textureRect.topRight());
qSetTex(v + 1, textureRect.topLeft());
qSetTex(v + 2, textureRect.bottomRight());
qSetTex(v + 3, textureRect.bottomLeft());
break;
}
if (m_frameTransformation.mirroredHorizontallyAfterRotation) {
qSwapTex(v + 0, v + 2);
qSwapTex(v + 1, v + 3);
}
if (!geometry())
setGeometry(g);
markDirty(DirtyGeometry);
setSubtitleGeometry();
}
const QVideoFrameTexturePoolPtr &QSGVideoNode::texturePool() const
{
return m_material->m_texturePool;
}
QT_END_NAMESPACE