Revision control
Copy as Markdown
// Copyright (C) 2026 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qohosvideooutput_p.h"
#include "qohosglobal_p.h"
#include "qohossurfaceimage_p.h"
#include <private/qhwvideobuffer_p.h>
#include <private/qplatformvideosink_p.h>
#include <private/qvideoframe_p.h>
#include <private/qvideoframeconverter_p.h>
#include <rhi/qrhi.h>
#include <rhi/qrhi_platform.h>
#include <QtMultimedia/qvideoframe.h>
#include <QtMultimedia/qvideoframeformat.h>
#include <QtMultimedia/qvideosink.h>
#include <QtMultimedia/qtvideo.h>
#include <QtGui/qmatrix4x4.h>
#include <QtGui/qopenglcontext.h>
#include <QtGui/qoffscreensurface.h>
#include <QtGui/qguiapplication.h>
#include <QtGui/qscreen.h>
#include <QtCore/qfile.h>
#include <QtCore/qloggingcategory.h>
#include <atomic>
#include <QtCore/qpointer.h>
#include <QtCore/qthread.h>
QT_BEGIN_NAMESPACE
namespace {
const float g_quad[] = {
-1.f, -1.f, 0.f, 0.f,
-1.f, 1.f, 0.f, 1.f,
1.f, 1.f, 1.f, 1.f,
1.f, -1.f, 1.f, 0.f
};
class QOhosVideoFrameTextures : public QVideoFrameTextures
{
public:
QOhosVideoFrameTextures(QRhi *rhi, QSize size, quint64 handle)
{
m_tex.reset(rhi->newTexture(QRhiTexture::RGBA8, size, 1));
m_tex->createFrom({ handle, 0 });
}
QRhiTexture *texture(uint plane) const override { return plane == 0 ? m_tex.get() : nullptr; }
private:
std::unique_ptr<QRhiTexture> m_tex;
};
class QOhosTextureVideoBuffer final : public QHwVideoBuffer
{
public:
QOhosTextureVideoBuffer(std::unique_ptr<QRhiTexture> tex, const QSize &size,
std::weak_ptr<QRhi> producerRhi, QPointer<QObject> producer,
QPointer<QOpenGLContext> producerContext)
: m_size(size)
, m_tex(std::move(tex))
, m_producerRhi(std::move(producerRhi))
, m_producer(std::move(producer))
, m_producerContext(std::move(producerContext))
{
}
MapData map(QVideoFrame::MapMode mode) override
{
MapData data;
if (m_mapMode != QVideoFrame::NotMapped || mode != QVideoFrame::ReadOnly)
return data;
m_mapMode = QVideoFrame::ReadOnly;
if (m_image.isNull())
m_image = readbackOnProducerThread();
if (m_image.isNull()) {
m_mapMode = QVideoFrame::NotMapped;
return data;
}
data.planeCount = 1;
data.bytesPerLine[0] = m_image.bytesPerLine();
data.dataSize[0] = static_cast<int>(m_image.sizeInBytes());
data.data[0] = m_image.bits();
return data;
}
void unmap() override
{
m_image = {};
m_mapMode = QVideoFrame::NotMapped;
}
QVideoFrameTexturesUPtr mapTextures(QRhi &rhi) override
{
// The texture lives in the producer (texture-thread) GL context. Sampling
// requires the caller's RHI to share that GL context's resources. The main
// window RHI does (we created the producer with shareContext = main rhi
// context). A worker thread's QThreadLocal RHI does not — so report no
// textures and let qImageFromVideoFrame fall back to CPU mapping.
if (!isCompatibleRhi(rhi))
return {};
return std::make_unique<QOhosVideoFrameTextures>(&rhi, m_size,
m_tex->nativeTexture().object);
}
private:
bool isCompatibleRhi(QRhi &rhi) const
{
if (rhi.backend() != QRhi::OpenGLES2)
return false;
if (!m_producerContext)
return false;
const auto *handles =
static_cast<const QRhiGles2NativeHandles *>(rhi.nativeHandles());
if (!handles || !handles->context)
return false;
return handles->context->shareGroup() == m_producerContext->shareGroup();
}
QImage readbackOnProducerThread() const
{
auto producerRhi = m_producerRhi.lock();
if (!producerRhi || !m_producer)
return {};
QImage out;
QRhi *rhi = producerRhi.get();
QRhiTexture *tex = m_tex.get();
QMetaObject::invokeMethod(
m_producer.data(),
[rhi, tex, &out]() {
QRhiReadbackResult result;
bool done = false;
result.completed = [&done] { done = true; };
QRhiCommandBuffer *cb = nullptr;
if (rhi->beginOffscreenFrame(&cb) != QRhi::FrameOpSuccess)
return;
QRhiResourceUpdateBatch *rub = rhi->nextResourceUpdateBatch();
rub->readBackTexture({ tex }, &result);
cb->resourceUpdate(rub);
rhi->endOffscreenFrame();
if (!done || result.data.isEmpty())
return;
QImage img(reinterpret_cast<const uchar *>(result.data.constData()),
result.pixelSize.width(), result.pixelSize.height(),
result.pixelSize.width() * 4, QImage::Format_RGBA8888);
out = img.copy();
},
Qt::BlockingQueuedConnection);
return out;
}
QSize m_size;
std::unique_ptr<QRhiTexture> m_tex;
QImage m_image;
QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped;
std::weak_ptr<QRhi> m_producerRhi;
QPointer<QObject> m_producer;
QPointer<QOpenGLContext> m_producerContext;
};
class TextureCopy
{
public:
TextureCopy(QRhi *rhi, QRhiTexture *externalTex) : m_rhi(rhi)
{
m_vertexBuffer.reset(m_rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer,
sizeof(g_quad)));
m_vertexBuffer->create();
m_uniformBuffer.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic,
QRhiBuffer::UniformBuffer, 160));
m_uniformBuffer->create();
m_sampler.reset(m_rhi->newSampler(QRhiSampler::Nearest, QRhiSampler::Nearest,
QRhiSampler::None, QRhiSampler::ClampToEdge,
QRhiSampler::ClampToEdge));
m_sampler->create();
m_srb.reset(m_rhi->newShaderResourceBindings());
m_srb->setBindings({
QRhiShaderResourceBinding::uniformBuffer(
0,
QRhiShaderResourceBinding::VertexStage
| QRhiShaderResourceBinding::FragmentStage,
m_uniformBuffer.get()),
QRhiShaderResourceBinding::sampledTexture(
1, QRhiShaderResourceBinding::FragmentStage, externalTex, m_sampler.get())
});
m_srb->create();
m_vertexShader = loadShader(
QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.vert.qsb"));
m_fragmentShader = loadShader(
QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.frag.qsb"));
}
std::unique_ptr<QRhiTexture> copyExternalTexture(QSize size, const QMatrix4x4 &externalTexMatrix);
private:
static QShader loadShader(const QString &name)
{
QFile f(name);
if (f.open(QIODevice::ReadOnly))
return QShader::fromSerialized(f.readAll());
return {};
}
QRhi *m_rhi{ nullptr };
std::unique_ptr<QRhiBuffer> m_vertexBuffer;
std::unique_ptr<QRhiBuffer> m_uniformBuffer;
std::unique_ptr<QRhiSampler> m_sampler;
std::unique_ptr<QRhiShaderResourceBindings> m_srb;
QShader m_vertexShader;
QShader m_fragmentShader;
};
std::unique_ptr<QRhiGraphicsPipeline> newGraphicsPipeline(QRhi *rhi,
QRhiShaderResourceBindings *srb,
QRhiRenderPassDescriptor *rpd,
QShader vs, QShader fs)
{
std::unique_ptr<QRhiGraphicsPipeline> gp(rhi->newGraphicsPipeline());
gp->setTopology(QRhiGraphicsPipeline::TriangleFan);
gp->setShaderStages({
{ QRhiShaderStage::Vertex, vs },
{ QRhiShaderStage::Fragment, fs }
});
QRhiVertexInputLayout layout;
layout.setBindings({ { 4 * sizeof(float) } });
layout.setAttributes({
{ 0, 0, QRhiVertexInputAttribute::Float2, 0 },
{ 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) }
});
gp->setVertexInputLayout(layout);
gp->setShaderResourceBindings(srb);
gp->setRenderPassDescriptor(rpd);
gp->create();
return gp;
}
std::unique_ptr<QRhiTexture>
TextureCopy::copyExternalTexture(QSize size, const QMatrix4x4 &externalTexMatrix)
{
std::unique_ptr<QRhiTexture> tex(
m_rhi->newTexture(QRhiTexture::RGBA8, size, 1, QRhiTexture::RenderTarget));
if (!tex->create()) {
qCWarning(qLcOhosMediaPlugin) << "Failed to create frame texture";
return {};
}
std::unique_ptr<QRhiTextureRenderTarget> renderTarget(
m_rhi->newTextureRenderTarget({ { tex.get() } }));
std::unique_ptr<QRhiRenderPassDescriptor> rpd(
renderTarget->newCompatibleRenderPassDescriptor());
renderTarget->setRenderPassDescriptor(rpd.get());
renderTarget->create();
QRhiResourceUpdateBatch *rub = m_rhi->nextResourceUpdateBatch();
rub->uploadStaticBuffer(m_vertexBuffer.get(), g_quad);
const QMatrix4x4 identity;
char *p = m_uniformBuffer->beginFullDynamicBufferUpdateForCurrentFrame();
memcpy(p, identity.constData(), 64);
memcpy(p + 64, externalTexMatrix.constData(), 64);
const float opacity = 1.0f;
memcpy(p + 64 + 64, &opacity, 4);
m_uniformBuffer->endFullDynamicBufferUpdateForCurrentFrame();
auto pipeline = newGraphicsPipeline(m_rhi, m_srb.get(), rpd.get(), m_vertexShader,
m_fragmentShader);
const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuffer.get(), 0);
QRhiCommandBuffer *cb = nullptr;
if (m_rhi->beginOffscreenFrame(&cb) != QRhi::FrameOpSuccess)
return {};
cb->beginPass(renderTarget.get(), Qt::transparent, { 1.0f, 0 }, rub);
cb->setGraphicsPipeline(pipeline.get());
cb->setViewport({ 0, 0, float(size.width()), float(size.height()) });
cb->setShaderResources(m_srb.get());
cb->setVertexInput(0, 1, &vbufBinding);
cb->draw(4);
cb->endPass();
m_rhi->endOffscreenFrame();
return tex;
}
} // namespace
class QOhosTextureThread : public QThread
{
Q_OBJECT
public:
QOhosTextureThread() : QThread() { }
~QOhosTextureThread() override
{
QMetaObject::invokeMethod(this, &QOhosTextureThread::tearDown, Qt::BlockingQueuedConnection);
quit();
wait();
}
void launch()
{
QThread::start();
moveToThread(this);
}
OHNativeWindow *nativeWindowBlocking(QRhi *rhi)
{
OHNativeWindow *window = nullptr;
QMetaObject::invokeMethod(
this, [&]() { window = ensureSurface(rhi); }, Qt::BlockingQueuedConnection);
return window;
}
QByteArray surfaceIdBlocking(QRhi *rhi)
{
QByteArray id;
QMetaObject::invokeMethod(
this,
[&]() {
ensureSurface(rhi);
if (m_surfaceImage)
id = m_surfaceImage->surfaceId();
},
Qt::BlockingQueuedConnection);
return id;
}
void setFrameSizeBlocking(QSize size)
{
QMetaObject::invokeMethod(
this, [&]() { m_size = size; }, Qt::BlockingQueuedConnection);
}
public Q_SLOTS:
void tearDown()
{
m_surfaceImage.reset();
m_externalTexture.reset();
m_textureCopy.reset();
m_rhi.reset();
m_shareContext = nullptr;
}
// Sampled on the GUI thread; QScreen must not be touched from this thread.
void setDisplayRotation(QtVideo::Rotation rotation)
{
m_displayRotation.store(rotation, std::memory_order_relaxed);
}
void onFrameAvailable(quint64 index)
{
if (!m_surfaceImage || m_surfaceImage->index() != index)
return;
if (!m_surfaceImage->updateTexImage())
return;
if (!m_textureCopy)
return;
// GL bottom-left to top-left origin flip.
static const QMatrix4x4 flipV(1.0f, 0.0f, 0.0f, 0.0f,
0.0f, -1.0f, 0.0f, 1.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f);
QMatrix4x4 matrix = m_surfaceImage->transformMatrix();
// The sensor transform swaps x/y on 90/270; transpose the target to match aspect.
const bool axesSwapped = std::abs(matrix(0, 0)) < std::abs(matrix(0, 1));
const QSize outSize = axesSwapped ? m_size.transposed() : m_size;
const QtVideo::Rotation displayRotation = m_displayRotation.load(std::memory_order_relaxed);
matrix *= flipV;
auto rgba = m_textureCopy->copyExternalTexture(outSize, matrix);
if (!rgba)
return;
QPointer<QOpenGLContext> ctx;
if (const auto *handles =
static_cast<const QRhiGles2NativeHandles *>(m_rhi->nativeHandles()))
ctx = handles->context;
auto buffer = std::make_unique<QOhosTextureVideoBuffer>(
std::move(rgba), outSize, std::weak_ptr<QRhi>(m_rhi),
QPointer<QObject>(this), ctx);
QVideoFrameFormat format(outSize, QVideoFrameFormat::Format_RGBA8888);
format.setRotation(displayRotation);
QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(buffer), format);
Q_EMIT newFrame(frame);
}
Q_SIGNALS:
void newFrame(const QVideoFrame &frame);
private:
OHNativeWindow *ensureSurface(QRhi *rhi)
{
const auto *nativeHandles =
rhi ? static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles())
: nullptr;
QOpenGLContext *shareContext = nativeHandles ? nativeHandles->context : nullptr;
// Re-use the existing surface only while it still shares the caller's
// GL context. For headless camera (rhi == nullptr) we accept any prior
// standalone surface.
const bool reuse = m_surfaceImage && m_surfaceImage->isValid()
&& (rhi ? !m_isHeadless && m_shareContext == shareContext : m_isHeadless);
if (reuse)
return m_surfaceImage->nativeWindow();
// Order matters: the texture and the copy pipeline outlive m_rhi otherwise.
tearDown();
// Share with the main RHI's GL context so consumers can sample the
// texture. For headless mode we create a standalone offscreen GLES2
// RHI without a share context — frames are produced but consumers
// (cameras, recorders) only need the native surface handle.
QRhiGles2InitParams params;
params.shareContext = shareContext;
params.fallbackSurface = QRhiGles2InitParams::newFallbackSurface();
m_isHeadless = (rhi == nullptr);
m_shareContext = shareContext;
m_rhi.reset(QRhi::create(QRhi::OpenGLES2, ¶ms));
if (!m_rhi) {
qCWarning(qLcOhosMediaPlugin) << "Failed to create offscreen GLES2 RHI";
return nullptr;
}
m_externalTexture.reset(
m_rhi->newTexture(QRhiTexture::RGBA8, m_size.isEmpty() ? QSize{ 1, 1 } : m_size, 1,
QRhiTexture::ExternalOES));
if (!m_externalTexture->create()) {
qCWarning(qLcOhosMediaPlugin) << "External OES texture create failed";
m_externalTexture.reset();
m_rhi.reset();
return nullptr;
}
const auto nativeTex = m_externalTexture->nativeTexture();
m_surfaceImage = std::make_unique<QOhosSurfaceImage>(uint32_t(nativeTex.object));
if (!m_surfaceImage->isValid()) {
m_surfaceImage.reset();
m_externalTexture.reset();
m_rhi.reset();
return nullptr;
}
const quint64 index = m_surfaceImage->index();
connect(m_surfaceImage.get(), &QOhosSurfaceImage::frameAvailable, this,
[this, index]() { onFrameAvailable(index); }, Qt::QueuedConnection);
m_textureCopy = std::make_unique<TextureCopy>(m_rhi.get(), m_externalTexture.get());
return m_surfaceImage->nativeWindow();
}
std::shared_ptr<QRhi> m_rhi;
std::unique_ptr<QRhiTexture> m_externalTexture;
std::unique_ptr<QOhosSurfaceImage> m_surfaceImage;
std::unique_ptr<TextureCopy> m_textureCopy;
QSize m_size{ 1, 1 };
std::atomic<QtVideo::Rotation> m_displayRotation{ QtVideo::Rotation::None };
QPointer<QOpenGLContext> m_shareContext;
bool m_isHeadless{ false };
};
QOhosVideoOutput::QOhosVideoOutput(QVideoSink *sink, ContentSource contentSource,
QObject *parent)
: QObject(parent), m_sink(sink), m_contentSource(contentSource)
{
m_textureThread = std::make_shared<QOhosTextureThread>();
connect(m_textureThread.get(), &QOhosTextureThread::newFrame, this,
&QOhosVideoOutput::onNewFrame, Qt::QueuedConnection);
m_textureThread->launch();
if (auto *p = sink ? sink->platformVideoSink() : nullptr) {
connect(p, &QPlatformVideoSink::rhiChanged, this, &QOhosVideoOutput::onRhiChanged);
}
if (m_contentSource == ContentSource::Camera) {
if (QScreen *screen = QGuiApplication::primaryScreen()) {
connect(screen, &QScreen::orientationChanged, this,
&QOhosVideoOutput::updateDisplayRotation);
}
updateDisplayRotation();
}
}
void QOhosVideoOutput::updateDisplayRotation()
{
QtVideo::Rotation rotation = QtVideo::Rotation::None;
if (const QScreen *screen = QGuiApplication::primaryScreen()) {
switch (screen->angleBetween(screen->orientation(), screen->nativeOrientation())) {
case 90:
rotation = QtVideo::Rotation::Clockwise90;
break;
case 180:
rotation = QtVideo::Rotation::Clockwise180;
break;
case 270:
rotation = QtVideo::Rotation::Clockwise270;
break;
default:
break;
}
}
m_textureThread->setDisplayRotation(rotation);
}
void QOhosVideoOutput::onRhiChanged()
{
if (!m_sink || !m_sink->rhi())
return;
if (m_surfaceCreatedWithoutRhi) {
tearDownTextureThread();
m_surfaceCreatedWithoutRhi = false;
}
Q_EMIT surfaceReady();
}
QOhosVideoOutput::~QOhosVideoOutput()
{
tearDownTextureThread();
}
void QOhosVideoOutput::tearDownTextureThread()
{
// Delivered frames own textures from the RHI that is about to go away.
if (m_sink)
m_sink->setVideoFrame({});
QMetaObject::invokeMethod(m_textureThread.get(), &QOhosTextureThread::tearDown,
Qt::BlockingQueuedConnection);
}
OHNativeWindow *QOhosVideoOutput::nativeWindow()
{
auto *rhi = m_sink ? m_sink->rhi() : nullptr;
if (!rhi) {
m_surfaceCreatedWithoutRhi = true;
} else if (m_surfaceCreatedWithoutRhi) {
tearDownTextureThread();
m_surfaceCreatedWithoutRhi = false;
}
return m_textureThread->nativeWindowBlocking(rhi);
}
QByteArray QOhosVideoOutput::surfaceId()
{
// The camera framework needs a surface even when there's no sink or the
// sink isn't bound to a window yet. Spin up an offscreen RHI internally so
// capture can proceed headless; surfaceReady will reattach once the sink
// gets a real RHI.
auto *rhi = m_sink ? m_sink->rhi() : nullptr;
if (!rhi)
m_surfaceCreatedWithoutRhi = true;
else if (m_surfaceCreatedWithoutRhi) {
tearDownTextureThread();
m_surfaceCreatedWithoutRhi = false;
}
return m_textureThread->surfaceIdBlocking(rhi);
}
void QOhosVideoOutput::setVideoSize(const QSize &size)
{
if (m_videoSize == size || !size.isValid())
return;
m_videoSize = size;
m_textureThread->setFrameSizeBlocking(size);
if (m_sink) {
if (auto *p = m_sink->platformVideoSink())
p->setNativeSize(size);
}
}
void QOhosVideoOutput::onNewFrame(const QVideoFrame &frame)
{
if (m_sink)
m_sink->setVideoFrame(frame);
}
QT_END_NAMESPACE
#include "qohosvideooutput.moc"
#include "moc_qohosvideooutput_p.cpp"