Revision control
++ // Copyright (C) 2022 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++ // Qt-Security score:critical reason:data-parser
++
++ #include "qvideoframeconverter_p.h"
++ #include "qvideoframeconversionhelper_p.h"
++ #include "qvideoframeformat.h"
++ #include "qvideoframe_p.h"
++ #include "qmultimediautils_p.h"
++ #include "qthreadlocalrhi_p.h"
++ #include "qcachedvalue_p.h"
++
++ #include <QtCore/qcoreapplication.h>
++ #include <QtCore/qsize.h>
++ #include <QtCore/qhash.h>
++ #include <QtCore/qfile.h>
++ #include <QtGui/qimage.h>
++ #include <QtCore/qloggingcategory.h>
++
++ #include <QtMultimedia/private/qaudio_qspan_support_p.h>
++ #include <QtMultimedia/private/qmultimedia_ranges_p.h>
++ #include <QtMultimedia/private/qvideotexturehelper_p.h>
++
++ #include <rhi/qrhi.h>
++
++ #ifdef Q_OS_DARWIN
++ #include <QtCore/private/qcore_mac_p.h>
++ #endif
++
++ QT_BEGIN_NAMESPACE
++
++ namespace ranges = QtMultimediaPrivate::ranges;
++
++ Q_STATIC_LOGGING_CATEGORY(qLcVideoFrameConverter, "qt.multimedia.video.frameconverter")
++
++ // clang-format off
++ static constexpr float g_quad[] = {
++ // Rotation 0 CW
++ 1.f, -1.f, 1.f, 1.f,
++ 1.f, 1.f, 1.f, 0.f,
++ -1.f, -1.f, 0.f, 1.f,
++ -1.f, 1.f, 0.f, 0.f,
++ // Rotation 90 CW
++ 1.f, -1.f, 1.f, 0.f,
++ 1.f, 1.f, 0.f, 0.f,
++ -1.f, -1.f, 1.f, 1.f,
++ -1.f, 1.f, 0.f, 1.f,
++ // Rotation 180 CW
++ 1.f, -1.f, 0.f, 0.f,
++ 1.f, 1.f, 0.f, 1.f,
++ -1.f, -1.f, 1.f, 0.f,
++ -1.f, 1.f, 1.f, 1.f,
++ // Rotation 270 CW
++ 1.f, -1.f, 0.f, 1.f,
++ 1.f, 1.f, 1.f, 1.f,
++ -1.f, -1.f, 0.f, 0.f,
++ -1.f, 1.f, 1.f, 0.f,
++ };
++ // clang-format on
++
++ static bool pixelFormatHasAlpha(QVideoFrameFormat::PixelFormat format)
++ {
++ switch (format) {
++ case QVideoFrameFormat::Format_ARGB8888:
++ case QVideoFrameFormat::Format_ARGB8888_Premultiplied:
++ case QVideoFrameFormat::Format_BGRA8888:
++ case QVideoFrameFormat::Format_BGRA8888_Premultiplied:
++ case QVideoFrameFormat::Format_ABGR8888:
++ case QVideoFrameFormat::Format_RGBA8888:
++ case QVideoFrameFormat::Format_AYUV:
++ case QVideoFrameFormat::Format_AYUV_Premultiplied:
++ return true;
++ default:
++ return false;
++ }
++ };
++
++ static QShader ensureShader(const QString &name)
++ {
++ static QCachedValueMap<QString, QShader> shaderCache;
++
++ return shaderCache.ensure(name, [&name]() {
++ QFile f(name);
++ return f.open(QIODevice::ReadOnly) ? QShader::fromSerialized(f.readAll()) : QShader();
++ });
++ }
++
++ static void rasterTransform(QImage &image, VideoTransformation transformation)
++ {
++ QTransform t;
++ // QTransform composes so that the first call acts last on the point (like
++ // QPainter's nested coordinate systems), so scale() must precede rotate()
++ // here for the mirror to be applied after the rotation, as required by
++ // mirroredHorizontallyAfterRotation.
++ if (transformation.mirroredHorizontallyAfterRotation)
++ t.scale(-1., 1);
++ if (transformation.rotation != QtVideo::Rotation::None)
++ t.rotate(qreal(transformation.rotation));
++ if (!t.isIdentity())
++ image = image.transformed(t);
++ }
++
++ static void imageCleanupHandler(void *info)
++ {
++ QByteArray *imageData = reinterpret_cast<QByteArray *>(info);
++ delete imageData;
++ }
++
++ static bool updateTextures(QRhi *rhi,
++ std::unique_ptr<QRhiBuffer> &uniformBuffer,
++ std::unique_ptr<QRhiSampler> &textureSampler,
++ std::unique_ptr<QRhiShaderResourceBindings> &shaderResourceBindings,
++ std::unique_ptr<QRhiGraphicsPipeline> &graphicsPipeline,
++ std::unique_ptr<QRhiRenderPassDescriptor> &renderPass,
++ QVideoFrame &frame,
++ const QVideoFrameTexturesUPtr &videoFrameTextures)
++ {
++ auto format = frame.surfaceFormat();
++ auto pixelFormat = format.pixelFormat();
++
++ auto textureDesc = QVideoTextureHelper::textureDescription(pixelFormat);
++
++ QRhiShaderResourceBinding bindings[4];
++ auto *b = bindings;
++ *b++ = QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage,
++ uniformBuffer.get());
++ for (int i = 0; i < textureDesc->nplanes; ++i)
++ *b++ = QRhiShaderResourceBinding::sampledTexture(i + 1, QRhiShaderResourceBinding::FragmentStage,
++ videoFrameTextures->texture(i), textureSampler.get());
++ shaderResourceBindings->setBindings(bindings, b);
++ if (!shaderResourceBindings->create()) {
++ qCDebug(qLcVideoFrameConverter)
++ << Q_FUNC_INFO << ": failed to create shader resource bindings";
++ return false;
++ }
++
++ graphicsPipeline.reset(rhi->newGraphicsPipeline());
++ graphicsPipeline->setTopology(QRhiGraphicsPipeline::TriangleStrip);
++
++ QShader vs = ensureShader(QVideoTextureHelper::vertexShaderFileName(format));
++ if (!vs.isValid())
++ return false;
++
++ QShader fs = ensureShader(QVideoTextureHelper::fragmentShaderFileName(format, rhi));
++ if (!fs.isValid())
++ return false;
++
++ graphicsPipeline->setShaderStages({
++ { QRhiShaderStage::Vertex, vs },
++ { QRhiShaderStage::Fragment, fs }
++ });
++
++ QRhiVertexInputLayout inputLayout;
++ inputLayout.setBindings({
++ { 4 * sizeof(float) }
++ });
++ inputLayout.setAttributes({
++ { 0, 0, QRhiVertexInputAttribute::Float2, 0 },
++ { 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) }
++ });
++
++ graphicsPipeline->setVertexInputLayout(inputLayout);
++ graphicsPipeline->setShaderResourceBindings(shaderResourceBindings.get());
++ graphicsPipeline->setRenderPassDescriptor(renderPass.get());
++ if (!graphicsPipeline->create()) {
++ qCDebug(qLcVideoFrameConverter) << Q_FUNC_INFO << ": failed to create graphics pipeline";
++ return false;
++ }
++
++ return true;
++ }
++
++ QSpan<const uchar> jpegDataFromMappedVideoFrame(const QVideoFrame &mappedFrame)
++ {
++ Q_ASSERT(mappedFrame.isReadable());
++ Q_ASSERT(mappedFrame.pixelFormat() == QVideoFrameFormat::PixelFormat::Format_Jpeg);
++
++ QSpan<const uchar> jpegData{
++ mappedFrame.bits(0),
++ mappedFrame.mappedBytes(0),
++ };
++
++ using namespace QtMultimediaPrivate;
++
++ constexpr std::array<uchar, 2> soiMarker{ uchar(0xff), uchar(0xd8) };
++ if (!ranges::equal(take(jpegData, 2), soiMarker, std::equal_to<void>{})) {
++ qCDebug(qLcVideoFrameConverter)
++ << Q_FUNC_INFO << ": JPEG data does not start with SOI marker";
++ return {};
++ }
++
++ constexpr std::array<uchar, 2> eoiMarker{ uchar(0xff), uchar(0xd9) };
++
++ // some JPEG cameras contain extra data after the JPEG marker. If so, we drop it to make
++ // libjpeg happy.
++ if (!ranges::equal(jpegData.last(2), eoiMarker, std::equal_to<void>{})) {
++ qCDebug(qLcVideoFrameConverter)
++ << Q_FUNC_INFO << ": JPEG data does not end with EOI marker";
++
++ auto eoi_it = std::find_end(jpegData.begin(), jpegData.end(), std::begin(eoiMarker),
++ std::end(eoiMarker));
++ if (eoi_it == jpegData.end()) {
++ qCWarning(qLcVideoFrameConverter)
++ << Q_FUNC_INFO << ": JPEG data does not contain EOI marker";
++ return {};
++ };
++
++ const size_t newSize = std::distance(jpegData.begin(), eoi_it) + std::size(eoiMarker);
++ jpegData = jpegData.first(newSize);
++ }
++
++ return jpegData;
++ }
++
++ static QImage convertJPEG(const QVideoFrame &frame, const VideoTransformation &transform)
++ {
++ QVideoFrame varFrame = frame;
++ if (!varFrame.map(QVideoFrame::ReadOnly)) {
++ qCDebug(qLcVideoFrameConverter) << Q_FUNC_INFO << ": frame mapping failed";
++ return {};
++ }
++
++ auto unmap = std::optional(QScopeGuard([&] {
++ varFrame.unmap();
++ }));
++
++ const QSpan<const uchar> jpegData = jpegDataFromMappedVideoFrame(varFrame);
++ if (jpegData.isEmpty())
++ return QImage{};
++
++ QImage image = QImage::fromData(jpegData, "JPG");
++ unmap = std::nullopt; // Release unmap guard
++ rasterTransform(image, transform);
++ return image;
++ }
++
++ static QImage convertCPU(const QVideoFrame &frame, const VideoTransformation &transform)
++ {
++ VideoFrameConvertFunc convert = qConverterForFormat(frame.pixelFormat());
++ if (!convert) {
++ qCDebug(qLcVideoFrameConverter) << Q_FUNC_INFO << ": unsupported pixel format" << frame.pixelFormat();
++ return {};
++ } else {
++ QVideoFrame varFrame = frame;
++ if (!varFrame.map(QVideoFrame::ReadOnly)) {
++ qCDebug(qLcVideoFrameConverter) << Q_FUNC_INFO << ": frame mapping failed";
++ return {};
++ }
++ auto format = pixelFormatHasAlpha(varFrame.pixelFormat()) ? QImage::Format_ARGB32_Premultiplied : QImage::Format_RGB32;
++ QImage image = QImage(varFrame.width(), varFrame.height(), format);
++ convert(varFrame, image.bits());
++ varFrame.unmap();
++ rasterTransform(image, transform);
++ return image;
++ }
++ }
++
++ QImage qImageFromVideoFrame(const QVideoFrame &frame, bool forceCpu)
++ {
++ // by default, surface transformation is applied, as full transformation is used for presentation only
++ return qImageFromVideoFrame(frame, qNormalizedSurfaceTransformation(frame.surfaceFormat()),
++ forceCpu);
++ }
++
++ QImage qImageFromVideoFrame(const QVideoFrame &frame, const VideoTransformation &transformation,
++ bool forceCpu)
++ {
++ #ifdef Q_OS_DARWIN
++ QMacAutoReleasePool releasePool;
++ #endif
++
++ std::unique_ptr<QRhiRenderPassDescriptor> renderPass;
++ std::unique_ptr<QRhiBuffer> vertexBuffer;
++ std::unique_ptr<QRhiBuffer> uniformBuffer;
++ std::unique_ptr<QRhiTexture> targetTexture;
++ std::unique_ptr<QRhiTextureRenderTarget> renderTarget;
++ std::unique_ptr<QRhiSampler> textureSampler;
++ std::unique_ptr<QRhiShaderResourceBindings> shaderResourceBindings;
++ std::unique_ptr<QRhiGraphicsPipeline> graphicsPipeline;
++
++ if (frame.size().isEmpty() || frame.pixelFormat() == QVideoFrameFormat::Format_Invalid)
++ return {};
++
++ if (frame.pixelFormat() == QVideoFrameFormat::Format_Jpeg)
++ return convertJPEG(frame, transformation);
++
++ if (forceCpu) // For test purposes
++ return convertCPU(frame, transformation);
++
++ QRhi *rhi = nullptr;
++
++ if (QHwVideoBuffer *buffer = QVideoFramePrivate::hwBuffer(frame))
++ rhi = buffer->associatedCurrentThreadRhi();
++
++ if (!rhi) {
++ // TODO: if a case with more the one available preferred backends appears,
++ // e.g. vulkan vs opengl, then we should:
++ // 1. Implement QHwVideoBuffer::preferredRhiBackend
++ // 2. Implement a map backend=>rhi inside qEnsureThreadLocalRhi
++ rhi = qEnsureThreadLocalRhi(/*buffer->preferredRhiBackend()*/);
++ }
++
++ if (!rhi || rhi->isRecordingFrame())
++ return convertCPU(frame, transformation);
++
++ Q_ASSERT(rhi->thread()->isCurrentThread());
++
++ // Do conversion using shaders
++
++ const QSize frameSize = qRotatedFrameSize(frame.size(), frame.surfaceFormat().rotation());
++
++ vertexBuffer.reset(rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_quad)));
++ if (!vertexBuffer->create()) {
++ qCDebug(qLcVideoFrameConverter) << "Failed to create vertex buffer. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ uniformBuffer.reset(rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, sizeof(QVideoTextureHelper::UniformData)));
++ if (!uniformBuffer->create()) {
++ qCDebug(qLcVideoFrameConverter) << "Failed to create uniform buffer. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ textureSampler.reset(rhi->newSampler(QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
++ QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge));
++ if (!textureSampler->create()) {
++ qCDebug(qLcVideoFrameConverter)
++ << "Failed to create texture sampler. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ shaderResourceBindings.reset(rhi->newShaderResourceBindings());
++
++ targetTexture.reset(rhi->newTexture(QRhiTexture::RGBA8, frameSize, 1, QRhiTexture::RenderTarget));
++ if (!targetTexture->create()) {
++ qCDebug(qLcVideoFrameConverter) << "Failed to create target texture. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ renderTarget.reset(rhi->newTextureRenderTarget({ { targetTexture.get() } }));
++ renderPass.reset(renderTarget->newCompatibleRenderPassDescriptor());
++ renderTarget->setRenderPassDescriptor(renderPass.get());
++ if (!renderTarget->create()) {
++ qCDebug(qLcVideoFrameConverter) << "Failed to create render target. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ QRhiCommandBuffer *cb = nullptr;
++ QRhi::FrameOpResult r = rhi->beginOffscreenFrame(&cb);
++ if (r != QRhi::FrameOpSuccess) {
++ qCDebug(qLcVideoFrameConverter) << "Failed to set up offscreen frame. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ QRhiResourceUpdateBatch *rub = rhi->nextResourceUpdateBatch();
++ Q_ASSERT(rub);
++
++ rub->uploadStaticBuffer(vertexBuffer.get(), g_quad);
++
++ QVideoFrame frameTmp = frame;
++ QVideoFrameTexturesUPtr noOldTextures;
++ auto videoFrameTextures = QVideoTextureHelper::createTextures(frameTmp, *rhi, *rub, noOldTextures);
++ if (!videoFrameTextures) {
++ qCDebug(qLcVideoFrameConverter) << "Failed obtain textures. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ if (!updateTextures(rhi, uniformBuffer, textureSampler, shaderResourceBindings,
++ graphicsPipeline, renderPass, frameTmp, videoFrameTextures)) {
++ qCDebug(qLcVideoFrameConverter) << "Failed to update textures. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ float xScale = transformation.mirroredHorizontallyAfterRotation ? -1.0 : 1.0;
++ // Two independent effects can each flip the rendered quad vertically before it reaches the
++ // QImage we build from the QRhiReadbackResult below:
++ // - readback row order is inverted iff isYUpInFramebuffer() (true only for OpenGL)
++ // - g_quad is authored for Y-up NDC, so it renders vertically mirrored iff the backend's NDC
++ // is not Y-up (isYUpInNDC() false; true only for Vulkan, currently)
++ // A flip is needed iff exactly one of the two applies; they cancel out otherwise.
++ const bool flipForFramebufferReadback = rhi->isYUpInFramebuffer();
++ const bool flipForClipSpaceConvention = !rhi->isYUpInNDC();
++ float yScale = (flipForFramebufferReadback != flipForClipSpaceConvention) ? -1.f : 1.f;
++
++ QMatrix4x4 transform;
++ transform.scale(xScale, yScale);
++
++ QByteArray uniformData(sizeof(QVideoTextureHelper::UniformData), Qt::Uninitialized);
++ QVideoTextureHelper::updateUniformData(&uniformData, rhi, frame.surfaceFormat(), frame,
++ transform, 1.f);
++ rub->updateDynamicBuffer(uniformBuffer.get(), 0, uniformData.size(), uniformData.constData());
++
++ cb->beginPass(renderTarget.get(), Qt::black, { 1.0f, 0 }, rub);
++ cb->setGraphicsPipeline(graphicsPipeline.get());
++
++ cb->setViewport({ 0, 0, float(frameSize.width()), float(frameSize.height()) });
++ cb->setShaderResources(shaderResourceBindings.get());
++
++ const quint32 vertexOffset = quint32(sizeof(float)) * 16 * transformation.rotationIndex();
++ const QRhiCommandBuffer::VertexInput vbufBinding(vertexBuffer.get(), vertexOffset);
++ cb->setVertexInput(0, 1, &vbufBinding);
++ cb->draw(4);
++
++ QRhiReadbackDescription readDesc(targetTexture.get());
++ QRhiReadbackResult readResult;
++ bool readCompleted = false;
++
++ readResult.completed = [&readCompleted] { readCompleted = true; };
++
++ rub = rhi->nextResourceUpdateBatch();
++ rub->readBackTexture(readDesc, &readResult);
++
++ cb->endPass(rub);
++
++ rhi->endOffscreenFrame();
++
++ if (!readCompleted) {
++ qCDebug(qLcVideoFrameConverter) << "Failed to read back texture. Using CPU conversion.";
++ return convertCPU(frame, transformation);
++ }
++
++ QByteArray *imageData = new QByteArray(readResult.data);
++
++ return QImage(reinterpret_cast<const uchar *>(imageData->constData()),
++ readResult.pixelSize.width(), readResult.pixelSize.height(),
++ QImage::Format_RGBA8888_Premultiplied, imageCleanupHandler, imageData);
++ }
++
++ QImage videoFramePlaneAsImage(QVideoFrame &frame, int plane, QImage::Format targetFormat,
++ QSize targetSize)
++ {
++ if (plane >= frame.planeCount())
++ return {};
++
++ if (!frame.map(QVideoFrame::ReadOnly)) {
++ qWarning() << "Cannot map a video frame in ReadOnly mode!";
++ return {};
++ }
++
++ auto frameHandle = QVideoFramePrivate::handle(frame);
++
++ // With incrementing the reference counter, we share the mapped QVideoFrame
++ // with the target QImage. The function imageCleanupFunction is going to adopt
++ // the frameHandle by QVideoFrame and dereference it upon the destruction.
++ frameHandle->ref.ref();
++
++ auto imageCleanupFunction = [](void *data) {
++ QVideoFrame frame = reinterpret_cast<QVideoFramePrivate *>(data)->adoptThisByVideoFrame();
++ Q_ASSERT(frame.isMapped());
++ frame.unmap();
++ };
++
++ const auto bytesPerLine = frame.bytesPerLine(plane);
++ const auto height =
++ bytesPerLine ? qMin(targetSize.height(), frame.mappedBytes(plane) / bytesPerLine) : 0;
++
++ return QImage(reinterpret_cast<const uchar *>(frame.bits(plane)), targetSize.width(), height,
++ bytesPerLine, targetFormat, imageCleanupFunction, frameHandle);
++ }
++
++ QT_END_NAMESPACE
++