Revision control
++ // Copyright (C) 2021 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ #include "qvideowindow_p.h"
++ #include <QPlatformSurfaceEvent>
++ #include <qfile.h>
++ #include <qpainter.h>
++ #include <private/qguiapplication_p.h>
++ #include <private/qmemoryvideobuffer_p.h>
++ #include <private/qhwvideobuffer_p.h>
++ #include <private/qmultimediautils_p.h>
++ #include <private/qvideoframe_p.h>
++ #include <qpa/qplatformintegration.h>
++
++ QT_BEGIN_NAMESPACE
++
++ static QSurface::SurfaceType platformSurfaceType()
++ {
++ #if defined(Q_OS_DARWIN)
++ return QSurface::MetalSurface;
++ #elif defined (Q_OS_WIN)
++ return QSurface::Direct3DSurface;
++ #endif
++
++ auto *integration = QGuiApplicationPrivate::platformIntegration();
++
++ if (!integration->hasCapability(QPlatformIntegration::OpenGL))
++ return QSurface::RasterSurface;
++
++ if (QCoreApplication::testAttribute(Qt::AA_ForceRasterWidgets))
++ return QSurface::RasterSurface;
++
++ return QSurface::OpenGLSurface;
++ }
++
++ QVideoWindowPrivate::QVideoWindowPrivate(QVideoWindow *q)
++ : q(q),
++ m_sink(new QVideoSink)
++ {
++ Q_ASSERT(q);
++
++ if (QGuiApplicationPrivate::platformIntegration()->hasCapability(QPlatformIntegration::RhiBasedRendering)) {
++ auto surfaceType = ::platformSurfaceType();
++ q->setSurfaceType(surfaceType);
++ switch (surfaceType) {
++ case QSurface::RasterSurface:
++ case QSurface::OpenVGSurface:
++ default:
++ // can't use those surfaces, need to render in SW
++ m_graphicsApi = QRhi::Null;
++ break;
++ case QSurface::OpenGLSurface:
++ m_graphicsApi = QRhi::OpenGLES2;
++ break;
++ case QSurface::VulkanSurface:
++ m_graphicsApi = QRhi::Vulkan;
++ break;
++ case QSurface::MetalSurface:
++ m_graphicsApi = QRhi::Metal;
++ break;
++ case QSurface::Direct3DSurface:
++ m_graphicsApi = QRhi::D3D11;
++ break;
++ }
++ }
++
++ QObject::connect(m_sink.get(), &QVideoSink::videoFrameChanged, q, &QVideoWindow::setVideoFrame);
++ }
++
++ QVideoWindowPrivate::~QVideoWindowPrivate()
++ {
++ QObject::disconnect(m_sink.get(), &QVideoSink::videoFrameChanged,
++ q, &QVideoWindow::setVideoFrame);
++ }
++
++ static const float g_vw_quad[] = {
++ // 4 clockwise rotation of texture vertexes (the second pair)
++ // Rotation 0
++ -1.f, -1.f, 0.f, 0.f,
++ -1.f, 1.f, 0.f, 1.f,
++ 1.f, -1.f, 1.f, 0.f,
++ 1.f, 1.f, 1.f, 1.f,
++ // Rotation 90
++ -1.f, -1.f, 0.f, 1.f,
++ -1.f, 1.f, 1.f, 1.f,
++ 1.f, -1.f, 0.f, 0.f,
++ 1.f, 1.f, 1.f, 0.f,
++
++ // Rotation 180
++ -1.f, -1.f, 1.f, 1.f,
++ -1.f, 1.f, 1.f, 0.f,
++ 1.f, -1.f, 0.f, 1.f,
++ 1.f, 1.f, 0.f, 0.f,
++ // Rotation 270
++ -1.f, -1.f, 1.f, 0.f,
++ -1.f, 1.f, 0.f, 0.f,
++ 1.f, -1.f, 1.f, 1.f,
++ 1.f, 1.f, 0.f, 1.f
++ };
++
++ static QShader vwGetShader(const QString &name)
++ {
++ QFile f(name);
++ if (f.open(QIODevice::ReadOnly))
++ return QShader::fromSerialized(f.readAll());
++
++ return QShader();
++ }
++
++ void QVideoWindowPrivate::initRhi()
++ {
++ if (m_graphicsApi == QRhi::Null)
++ return;
++
++ [[maybe_unused]] QRhi::Flags rhiFlags = {};//QRhi::EnableDebugMarkers | QRhi::EnableProfiling;
++
++ #if QT_CONFIG(opengl)
++ if (m_graphicsApi == QRhi::OpenGLES2) {
++ m_fallbackSurface.reset(QRhiGles2InitParams::newFallbackSurface(q->format()));
++ QRhiGles2InitParams params;
++ params.fallbackSurface = m_fallbackSurface.get();
++ params.window = q;
++ params.format = q->format();
++ m_rhi.reset(QRhi::create(QRhi::OpenGLES2, ¶ms, rhiFlags));
++ }
++ #endif
++
++ #if QT_CONFIG(vulkan)
++ if (m_graphicsApi == QRhi::Vulkan) {
++ QRhiVulkanInitParams params;
++ params.inst = q->vulkanInstance();
++ params.window = q;
++ m_rhi.reset(QRhi::create(QRhi::Vulkan, ¶ms, rhiFlags));
++ }
++ #endif
++
++ #ifdef Q_OS_WIN
++ if (m_graphicsApi == QRhi::D3D11) {
++ QRhiD3D11InitParams params;
++ params.enableDebugLayer = true;
++ m_rhi.reset(QRhi::create(QRhi::D3D11, ¶ms, rhiFlags));
++ }
++ #endif
++
++ #if QT_CONFIG(metal)
++ if (m_graphicsApi == QRhi::Metal) {
++ QRhiMetalInitParams params;
++ m_rhi.reset(QRhi::create(QRhi::Metal, ¶ms, rhiFlags));
++ }
++ #endif
++ if (!m_rhi)
++ return;
++
++ m_swapChain.reset(m_rhi->newSwapChain());
++ m_swapChain->setWindow(q);
++ m_renderPass.reset(m_swapChain->newCompatibleRenderPassDescriptor());
++ m_swapChain->setRenderPassDescriptor(m_renderPass.get());
++
++ m_vertexBuf.reset(m_rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_vw_quad)));
++ m_vertexBuf->create();
++ m_vertexBufReady = false;
++
++ m_uniformBuf.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, sizeof(QVideoTextureHelper::UniformData)));
++ m_uniformBuf->create();
++
++ m_textureSampler.reset(m_rhi->newSampler(QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
++ QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge));
++ m_textureSampler->create();
++
++ m_shaderResourceBindings.reset(m_rhi->newShaderResourceBindings());
++ m_subtitleResourceBindings.reset(m_rhi->newShaderResourceBindings());
++
++ m_subtitleUniformBuf.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, sizeof(QVideoTextureHelper::UniformData)));
++ m_subtitleUniformBuf->create();
++ }
++
++ void QVideoWindowPrivate::setupGraphicsPipeline(QRhiGraphicsPipeline *pipeline, QRhiShaderResourceBindings *bindings, const QVideoFrameFormat &fmt)
++ {
++
++ pipeline->setTopology(QRhiGraphicsPipeline::TriangleStrip);
++ QShader vs = vwGetShader(QVideoTextureHelper::vertexShaderFileName(fmt));
++ Q_ASSERT(vs.isValid());
++ QShader fs = vwGetShader(QVideoTextureHelper::fragmentShaderFileName(
++ fmt, m_rhi.get(), m_swapChain->format()));
++ Q_ASSERT(fs.isValid());
++ pipeline->setShaderStages({
++ { QRhiShaderStage::Vertex, vs },
++ { QRhiShaderStage::Fragment, fs }
++ });
++ QRhiVertexInputLayout inputLayout;
++ inputLayout.setBindings({
++ { 4 * sizeof(float) }
++ });
++ inputLayout.setAttributes({
++ { 0, 0, QRhiVertexInputAttribute::Float2, 0 },
++ { 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) }
++ });
++ pipeline->setVertexInputLayout(inputLayout);
++ pipeline->setShaderResourceBindings(bindings);
++ pipeline->setRenderPassDescriptor(m_renderPass.get());
++ pipeline->create();
++ }
++
++ void QVideoWindowPrivate::updateTextures(QRhiResourceUpdateBatch *rub)
++ {
++ // We render a 1x1 black pixel when we don't have a video
++ if (!m_texturePool.currentFrame().isValid())
++ m_texturePool.setCurrentFrame(QVideoFramePrivate::createFrame(
++ std::make_unique<QMemoryVideoBuffer>(QByteArray{ 4, 0 }, 4),
++ QVideoFrameFormat(QSize(1, 1), QVideoFrameFormat::Format_RGBA8888)));
++
++ if (!m_texturePool.texturesDirty())
++ return;
++
++ QVideoFrameTextures *textures = m_texturePool.updateTextures(*m_rhi, *rub);
++ if (!textures)
++ return;
++
++ QRhiShaderResourceBinding bindings[4];
++ auto *b = bindings;
++ *(b++) = QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage,
++ m_uniformBuf.get());
++
++ auto fmt = m_texturePool.currentFrame().surfaceFormat();
++ auto textureDesc = QVideoTextureHelper::textureDescription(fmt.pixelFormat());
++
++ for (int i = 0; i < textureDesc->nplanes; ++i)
++ (*b++) = QRhiShaderResourceBinding::sampledTexture(
++ i + 1, QRhiShaderResourceBinding::FragmentStage, textures->texture(i),
++ m_textureSampler.get());
++ m_shaderResourceBindings->setBindings(bindings, b);
++ m_shaderResourceBindings->create();
++
++ if (fmt != format) {
++ format = fmt;
++ if (!m_graphicsPipeline)
++ m_graphicsPipeline.reset(m_rhi->newGraphicsPipeline());
++
++ setupGraphicsPipeline(m_graphicsPipeline.get(), m_shaderResourceBindings.get(), format);
++ }
++ }
++
++ void QVideoWindowPrivate::updateSubtitle(QRhiResourceUpdateBatch *rub, const QSize &frameSize)
++ {
++ m_subtitleDirty = false;
++ m_hasSubtitle = !m_texturePool.currentFrame().subtitleText().isEmpty();
++ if (!m_hasSubtitle)
++ return;
++
++ m_subtitleLayout.update(frameSize, m_texturePool.currentFrame().subtitleText());
++ QSize size = m_subtitleLayout.bounds.size().toSize();
++
++ QImage img = m_subtitleLayout.toImage();
++
++ m_subtitleTexture.reset(m_rhi->newTexture(QRhiTexture::RGBA8, size));
++ m_subtitleTexture->create();
++ rub->uploadTexture(m_subtitleTexture.get(), img);
++
++ QRhiShaderResourceBinding bindings[2];
++
++ bindings[0] = QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage,
++ m_subtitleUniformBuf.get());
++
++ bindings[1] = QRhiShaderResourceBinding::sampledTexture(1, QRhiShaderResourceBinding::FragmentStage,
++ m_subtitleTexture.get(), m_textureSampler.get());
++ m_subtitleResourceBindings->setBindings(bindings, bindings + 2);
++ m_subtitleResourceBindings->create();
++
++ if (!m_subtitlePipeline) {
++ m_subtitlePipeline.reset(m_rhi->newGraphicsPipeline());
++
++ QRhiGraphicsPipeline::TargetBlend blend;
++ blend.enable = true;
++ m_subtitlePipeline->setTargetBlends({ blend });
++ setupGraphicsPipeline(m_subtitlePipeline.get(), m_subtitleResourceBindings.get(), QVideoFrameFormat(QSize(1, 1), QVideoFrameFormat::Format_RGBA8888));
++ }
++ }
++
++ void QVideoWindowPrivate::init()
++ {
++ if (initialized)
++ return;
++ initialized = true;
++
++ initRhi();
++
++ if (!m_rhi)
++ backingStore = new QBackingStore(q);
++ else
++ m_sink->setRhi(m_rhi.get());
++ }
++
++ void QVideoWindowPrivate::resizeSwapChain()
++ {
++ m_hasSwapChain = m_swapChain->createOrResize();
++ }
++
++ void QVideoWindowPrivate::releaseSwapChain()
++ {
++ if (m_hasSwapChain) {
++ m_hasSwapChain = false;
++ m_swapChain->destroy();
++ }
++ }
++
++ void QVideoWindowPrivate::render()
++ {
++ if (!initialized)
++ init();
++
++ if (!q->isExposed() || !isExposed)
++ return;
++
++ QRect rect(0, 0, q->width(), q->height());
++
++ if (backingStore) {
++ if (backingStore->size() != q->size())
++ backingStore->resize(q->size());
++
++ backingStore->beginPaint(rect);
++
++ QPaintDevice *device = backingStore->paintDevice();
++ if (!device)
++ return;
++ QPainter painter(device);
++
++ QVideoFrame frame = m_texturePool.currentFrame();
++ frame.paint(&painter, rect, { Qt::black, aspectRatioMode });
++ painter.end();
++
++ backingStore->endPaint();
++ backingStore->flush(rect);
++ return;
++ }
++
++ const VideoTransformation frameTransformation =
++ qNormalizedFrameTransformation(m_texturePool.currentFrame().surfaceFormat());
++ const QSize frameSize = qRotatedFramePresentationSize(m_texturePool.currentFrame());
++ const QSize scaled = frameSize.scaled(rect.size(), aspectRatioMode);
++ QRect videoRect = QRect(QPoint(0, 0), scaled);
++ videoRect.moveCenter(rect.center());
++ QRect subtitleRect = videoRect.intersected(rect);
++
++ if (!m_hasSwapChain || (m_swapChain->currentPixelSize() != m_swapChain->surfacePixelSize()))
++ resizeSwapChain();
++
++ const auto requiredSwapChainFormat =
++ qGetRequiredSwapChainFormat(m_texturePool.currentFrame().surfaceFormat());
++ if (qShouldUpdateSwapChainFormat(m_swapChain.get(), requiredSwapChainFormat)) {
++ releaseSwapChain();
++ m_swapChain->setFormat(requiredSwapChainFormat);
++ resizeSwapChain();
++ }
++
++ if (!m_hasSwapChain)
++ return;
++
++ QRhi::FrameOpResult r = m_rhi->beginFrame(m_swapChain.get());
++
++ if (r == QRhi::FrameOpSwapChainOutOfDate) {
++ resizeSwapChain();
++ if (!m_hasSwapChain)
++ return;
++ r = m_rhi->beginFrame(m_swapChain.get());
++ }
++ if (r != QRhi::FrameOpSuccess) {
++ qWarning("beginFrame failed with %d, retry", r);
++ q->requestUpdate();
++ return;
++ }
++
++ QRhiResourceUpdateBatch *rub = m_rhi->nextResourceUpdateBatch();
++
++ if (!m_vertexBufReady) {
++ m_vertexBufReady = true;
++ rub->uploadStaticBuffer(m_vertexBuf.get(), g_vw_quad);
++ }
++
++ updateTextures(rub);
++
++ if (m_subtitleDirty || m_subtitleLayout.videoSize != subtitleRect.size())
++ updateSubtitle(rub, subtitleRect.size());
++
++ const float mirrorFrame = frameTransformation.mirroredHorizontallyAfterRotation ? -1.f : 1.f;
++ const float xscale = mirrorFrame * float(videoRect.width()) / float(rect.width());
++ const float yscale = -1.f * float(videoRect.height()) / float(rect.height());
++
++ QMatrix4x4 transform;
++ transform.scale(xscale, yscale);
++
++ float maxNits = 100;
++ if (m_swapChain->format() == QRhiSwapChain::HDRExtendedSrgbLinear) {
++ auto info = m_swapChain->hdrInfo();
++ if (info.limitsType == QRhiSwapChainHdrInfo::ColorComponentValue)
++ maxNits = 100 * info.limits.colorComponentValue.maxColorComponentValue;
++ else
++ maxNits = info.limits.luminanceInNits.maxLuminance;
++ }
++
++ QByteArray uniformData;
++ QVideoTextureHelper::updateUniformData(&uniformData, m_rhi.get(),
++ m_texturePool.currentFrame().surfaceFormat(),
++ m_texturePool.currentFrame(), transform, 1.f, maxNits);
++ rub->updateDynamicBuffer(m_uniformBuf.get(), 0, uniformData.size(), uniformData.constData());
++
++ if (m_hasSubtitle) {
++ QMatrix4x4 st;
++ st.translate(0, -2.f * (float(m_subtitleLayout.bounds.center().y()) + float(subtitleRect.top()))/ float(rect.height()) + 1.f);
++ st.scale(float(m_subtitleLayout.bounds.width())/float(rect.width()),
++ -1.f * float(m_subtitleLayout.bounds.height())/float(rect.height()));
++
++ QByteArray uniformData;
++ QVideoFrameFormat fmt(m_subtitleLayout.bounds.size().toSize(), QVideoFrameFormat::Format_ARGB8888);
++ QVideoTextureHelper::updateUniformData(&uniformData, m_rhi.get(), fmt, QVideoFrame(), st,
++ 1.f);
++ rub->updateDynamicBuffer(m_subtitleUniformBuf.get(), 0, uniformData.size(), uniformData.constData());
++ }
++
++ QRhiCommandBuffer *cb = m_swapChain->currentFrameCommandBuffer();
++ cb->beginPass(m_swapChain->currentFrameRenderTarget(), Qt::black, { 1.0f, 0 }, rub);
++ cb->setGraphicsPipeline(m_graphicsPipeline.get());
++ auto size = m_swapChain->currentPixelSize();
++ cb->setViewport({ 0, 0, float(size.width()), float(size.height()) });
++ cb->setShaderResources(m_shaderResourceBindings.get());
++
++ const quint32 vertexOffset = quint32(sizeof(float)) * 16 * frameTransformation.rotationIndex();
++ const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuf.get(), vertexOffset);
++ cb->setVertexInput(0, 1, &vbufBinding);
++ cb->draw(4);
++
++ if (m_hasSubtitle) {
++ cb->setGraphicsPipeline(m_subtitlePipeline.get());
++ cb->setShaderResources(m_subtitleResourceBindings.get());
++ const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuf.get(), 0);
++ cb->setVertexInput(0, 1, &vbufBinding);
++ cb->draw(4);
++ }
++
++ cb->endPass();
++
++ m_rhi->endFrame(m_swapChain.get());
++
++ m_texturePool.onFrameEndInvoked();
++ }
++
++ /*!
++ \class QVideoWindow
++ \internal
++ */
++ QVideoWindow::QVideoWindow(QScreen *screen)
++ : QWindow(screen)
++ , d(new QVideoWindowPrivate(this))
++ {
++ }
++
++ QVideoWindow::QVideoWindow(QWindow *parent)
++ : QWindow(parent)
++ , d(new QVideoWindowPrivate(this))
++ {
++ }
++
++ QVideoWindow::~QVideoWindow() = default;
++
++ QVideoSink *QVideoWindow::videoSink() const
++ {
++ return d->m_sink.get();
++ }
++
++ Qt::AspectRatioMode QVideoWindow::aspectRatioMode() const
++ {
++ return d->aspectRatioMode;
++ }
++
++ void QVideoWindow::setAspectRatioMode(Qt::AspectRatioMode mode)
++ {
++ if (d->aspectRatioMode == mode)
++ return;
++ d->aspectRatioMode = mode;
++ Q_EMIT aspectRatioModeChanged(mode);
++ }
++
++ bool QVideoWindow::event(QEvent *e)
++ {
++ switch (e->type()) {
++ case QEvent::UpdateRequest:
++ d->render();
++ return true;
++
++ case QEvent::PlatformSurface:
++ // this is the proper time to tear down the swapchain (while the native window and surface are still around)
++ if (static_cast<QPlatformSurfaceEvent *>(e)->surfaceEventType() == QPlatformSurfaceEvent::SurfaceAboutToBeDestroyed) {
++ d->releaseSwapChain();
++ d->isExposed = false;
++ }
++ break;
++ case QEvent::Expose:
++ d->isExposed = isExposed();
++ if (d->isExposed)
++ d->render();
++ return true;
++
++ default:
++ break;
++ }
++
++ return QWindow::event(e);
++ }
++
++ void QVideoWindow::resizeEvent(QResizeEvent *resizeEvent)
++ {
++ if (!d->backingStore)
++ return;
++ if (!d->initialized)
++ d->init();
++ d->backingStore->resize(resizeEvent->size());
++ }
++
++ void QVideoWindow::setVideoFrame(const QVideoFrame &frame)
++ {
++ if (d->m_texturePool.currentFrame().subtitleText() != frame.subtitleText())
++ d->m_subtitleDirty = true;
++ d->m_texturePool.setCurrentFrame(frame);
++ if (d->isExposed)
++ requestUpdate();
++ }
++
++ QT_END_NAMESPACE
++
++ #include "moc_qvideowindow_p.cpp"