qwasmvideoframegrabber.cpp

++ // Copyright (C) 2026 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++

++ #include "qwasmvideoframegrabber_p.h"

++

++ #include "qwasmvideooutput_p.h"

++ #include "qwasmgltexturevideobuffer_p.h"

++

++ #include <QtMultimedia/qvideosink.h>

++ #include <QtMultimedia/private/qmemoryvideobuffer_p.h>

++ #include <QtMultimedia/private/qvideoframe_p.h>

++ #include <QtGui/qopenglcontext.h>

++ #include <QtGui/rhi/qrhi_platform.h>

++ #include <QtCore/qdebug.h>

++ #include <QtCore/private/qstdweb_p.h>

++

++ #include <emscripten/emscripten.h>

++ #include <emscripten/em_js.h>

++ #include <emscripten/html5.h>

++ #include <emscripten/val.h>

++

++ #include <GLES2/gl2.h>

++

++ QT_BEGIN_NAMESPACE

++

++ using namespace emscripten;

++ using namespace Qt::Literals;

++

++ // EM_JS pointer arguments are BigInt on wasm64, Number() makes them usable on both targets.

++

++ // Upload the current video frame to the already-bound TEXTURE_2D.

++ EM_JS(void, em_texImage2DFromVideo, (const char *videoId, int *widthOut, int *heightOut), {

++     videoId = Number(videoId);

++     widthOut = Number(widthOut);

++     heightOut = Number(heightOut);

++     var gl = GL.currentContext.GLctx;

++     var video = document.getElementById(UTF8ToString(videoId));

++     if (!video) { return; }

++     var frame;

++     try { frame = new VideoFrame(video); } catch(e) { return; }

++     HEAP32[widthOut / 4] = frame.displayWidth;

++     HEAP32[heightOut / 4] = frame.displayHeight;

++     gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);

++     frame.close();

++ });

++

++ QWasmVideoFrameGrabber::QWasmVideoFrameGrabber(QWasmVideoOutput *videoOutput)

++     : m_videoOutput(videoOutput)

++ {

++ }

++

++ void QWasmVideoFrameGrabber::detectWebGLContext()

++ {

++     m_glContextHandle = 0;

++     m_hasWebGLContext = false;

++

++     QRhi *rhi = m_videoOutput->m_wasmSink ? m_videoOutput->m_wasmSink->rhi() : nullptr;

++     if (!rhi || rhi->backend() != QRhi::OpenGLES2)

++         return;

++

++     const auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());

++     if (!nativeHandles || !nativeHandles->context)

++         return;

++     QOpenGLContext *openGLContext = nativeHandles->context;

++

++     auto tryGetHandleFromSurface = [&]() -> bool {

++         QSurface *surface = openGLContext->surface();

++         if (!surface)

++             return false;

++         // QWasmOpenGLContext::makeCurrent() ends in

++         // emscripten_webgl_make_context_current(), so the handle we want is the

++         // one that is current afterwards.

++         if (QOpenGLContext::currentContext() != openGLContext

++             && !openGLContext->makeCurrent(surface))

++             return false;

++         m_glContextHandle = emscripten_webgl_get_current_context();

++         m_hasWebGLContext = (m_glContextHandle != 0);

++         return m_hasWebGLContext;

++     };

++

++     if (!tryGetHandleFromSurface())

++         qWarning() << Q_FUNC_INFO << "could not determine the WebGL context for the current RHI";

++ }

++

++ // software path, copies VideoFrame data into a QMemoryVideoBuffer

++ void QWasmVideoFrameGrabber::processVideoFrame()

++ {

++     emscripten::val videoElement = m_videoOutput->currentVideoElement();

++

++     // The VideoFrame constructor throws InvalidStateError when the browser compositor

++     // has not yet committed the first decoded frame, even if readyState == 4 and

++     // videoWidth > 0. Use a JS try-catch so the exception does not propagate into

++     // the wasm runtime and abort the application.

++     emscripten::val oneVideoFrame = emscripten::val::take_ownership(

++             static_cast<EM_VAL>(EM_ASM_PTR({

++                 try {

++                     return Emval.toHandle(new VideoFrame(Emval.toValue($0)));

++                 } catch(e) {

++                     return Emval.toHandle(null);

++                 }

++             }, videoElement.as_handle())));

++

++     if (oneVideoFrame.isNull() || oneVideoFrame.isUndefined()) {

++         qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "VideoFrame not ready yet, skipping";

++         return;

++     }

++

++     emscripten::val options = emscripten::val::object();

++     emscripten::val rectOptions = emscripten::val::object();

++

++     int displayWidth = oneVideoFrame["displayWidth"].as<int>();

++     int displayHeight = oneVideoFrame["displayHeight"].as<int>();

++

++     rectOptions.set("width", displayWidth);

++     rectOptions.set("height", displayHeight);

++     options.set("rect", rectOptions);

++

++     emscripten::val frameBytesAllocationSize = oneVideoFrame.call<emscripten::val>("allocationSize", options);

++     emscripten::val frameBuffer =

++             emscripten::val::global("Uint8Array").new_(frameBytesAllocationSize);

++     QWasmVideoOutput *videoOutput = m_videoOutput;

++

++     qstdweb::PromiseCallbacks copyToCallback;

++     copyToCallback.thenFunc = [videoOutput, oneVideoFrame, frameBuffer,

++                                 displayWidth, displayHeight]

++             (emscripten::val frameLayout)

++     {

++         if (frameLayout.isNull() || frameLayout.isUndefined()) {

++             qCDebug(qWasmMediaVideoOutput) << "theres no frameLayout";

++             return;

++         }

++

++         // frameBuffer now has a new frame, send to Qt

++         const QSize frameSize(displayWidth, displayHeight);

++

++         QByteArray frameBytes = QByteArray::fromEcmaUint8Array(frameBuffer);

++

++         QVideoFrameFormat::PixelFormat pixelFormat =

++                 fromJsPixelFormat(oneVideoFrame["format"].as<std::string>());

++         if (pixelFormat == QVideoFrameFormat::Format_Invalid)

++             pixelFormat = QVideoFrameFormat::Format_RGBA8888;

++         QVideoFrameFormat frameFormat = QVideoFrameFormat(frameSize, pixelFormat);

++

++         if (videoOutput->m_useCameraRotation)

++             frameFormat.setRotation(videoOutput->m_rotateBy);

++         if (videoOutput->m_streamFrameRate > 0)

++             frameFormat.setStreamFrameRate(videoOutput->m_streamFrameRate);

++         auto buffer = std::make_unique<QMemoryVideoBuffer>(

++                 std::move(frameBytes),

++                 frameLayout[0]["stride"].as<int>());

++

++         QVideoFrame vFrame =

++                 QVideoFramePrivate::createFrame(std::move(buffer), std::move(frameFormat));

++

++         if (!videoOutput->m_wasmSink) {

++             qWarning() << "ERROR ALERT!! video sink not set";

++             return;

++         }

++         videoOutput->m_wasmSink->setVideoFrame(vFrame);

++         oneVideoFrame.call<emscripten::val>("close");

++     };

++     copyToCallback.catchFunc = [oneVideoFrame](emscripten::val error)

++     {

++         qCDebug(qWasmMediaVideoOutput) << "copyTo error"

++                                << QString::fromStdString(error["name"].as<std::string>())

++                                << QString::fromStdString(error["message"].as<std::string>());

++         oneVideoFrame.call<emscripten::val>("close");

++     };

++

++     qstdweb::Promise::make(oneVideoFrame, u"copyTo"_s, std::move(copyToCallback), frameBuffer, options);

++ }

++

++ // zero-readback path, uploads the VideoFrame straight into a GL texture

++ void QWasmVideoFrameGrabber::processWebGLVideoFrame()

++ {

++     emscripten_webgl_make_context_current(m_glContextHandle);

++

++     GLuint rawTextureId = 0;

++     glGenTextures(1, &rawTextureId);

++     QGlTextureHandle textureHandle{ rawTextureId };

++

++     glBindTexture(GL_TEXTURE_2D, textureHandle.get());

++

++     int width = 0, height = 0;

++     em_texImage2DFromVideo(m_videoOutput->m_videoSurfaceId.c_str(), &width, &height);

++

++     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);

++     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);

++     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

++     glBindTexture(GL_TEXTURE_2D, 0);

++

++     if (!textureHandle || width == 0 || height == 0) {

++         qCWarning(qWasmMediaVideoOutput) << "VideoFrame upload failed";

++         return;

++     }

++

++     std::unique_ptr<QHwVideoBuffer> hwBuffer =

++             std::make_unique<QWasmGLTextureVideoBuffer>(

++                     std::move(textureHandle), QSize(width, height),

++                     m_glContextHandle,

++                     m_videoOutput->m_wasmSink ? m_videoOutput->m_wasmSink->rhi() : nullptr);

++

++     QVideoFrameFormat frameFormat(QSize(width, height), QVideoFrameFormat::Format_RGBA8888);

++     if (m_videoOutput->m_streamFrameRate > 0)

++         frameFormat.setStreamFrameRate(m_videoOutput->m_streamFrameRate);

++     QVideoFrame vFrame =

++             QVideoFramePrivate::createFrame(std::move(hwBuffer), std::move(frameFormat));

++

++     m_videoOutput->m_wasmSink->setVideoFrame(vFrame);

++ }

++

++ void QWasmVideoFrameGrabber::startFrameLoop()

++ {

++     if (!m_webGLContextChecked) {

++         m_webGLContextChecked = true;

++         detectWebGLContext();

++     }

++

++     m_videoOutput->detectIosCameraRotation();

++

++     // Single-shot callback: re-registers each frame so multiple QWasmVideoOutput

++     // instances can coexist. emscripten_request_animation_frame_loop allows only one

++     // active loop globally and would cancel another instance.

++     static EM_BOOL (*frame)(double, void *) = [](double frameTime, void *context) -> EM_BOOL {

++

++         Q_UNUSED(frameTime);

++

++         QWasmVideoFrameGrabber *frameGrabber = reinterpret_cast<QWasmVideoFrameGrabber *>(context);

++         QWasmVideoOutput *videoOutput = frameGrabber ? frameGrabber->m_videoOutput : nullptr;

++         if (!videoOutput || videoOutput->m_isStopped) {

++             qCWarning(qWasmMediaVideoOutput) << "frame loop exit: isStopped=" << (videoOutput ? videoOutput->m_isStopped : true)

++                                              << "mode=" << (videoOutput ? videoOutput->m_currentVideoMode : -1);

++             return false;

++         }

++

++         if (videoOutput->m_currentVideoMode == QWasmVideoOutput::VideoDisplay

++             && videoOutput->m_currentMediaStatus != QWasmVideoOutput::MediaStatus::LoadedMedia) {

++             emscripten_request_animation_frame(frame, context);

++             return true;

++         }

++

++         emscripten::val videoElement = videoOutput->currentVideoElement();

++         if (videoElement.isNull() || videoElement.isUndefined()) {

++             qCWarning(qWasmMediaVideoOutput) << "frame loop exit: video element null, mode=" << videoOutput->m_currentVideoMode;

++             return false;

++         }

++

++         if (videoElement["paused"].as<bool>() || videoElement["ended"].as<bool>()

++             || videoElement["readyState"].as<int>() < 2) {

++             qCDebug(qWasmMediaVideoOutput) << "frame loop waiting: mode=" << videoOutput->m_currentVideoMode

++                                            << "paused=" << videoElement["paused"].as<bool>()

++                                            << "ended=" << videoElement["ended"].as<bool>()

++                                            << "readyState=" << videoElement["readyState"].as<int>();

++             emscripten_request_animation_frame(frame, context);

++             return true;

++         }

++

++         if (frameGrabber->m_glContextHandle)

++             frameGrabber->processWebGLVideoFrame();

++         else

++             frameGrabber->processVideoFrame();

++

++         emscripten_request_animation_frame(frame, context);

++         return true;

++     };

++     if ((!m_videoOutput->m_isStopped

++          && m_videoOutput->m_video["className"].as<std::string>() == "Camera"

++          && m_videoOutput->m_cameraIsReady)

++         || (!m_videoOutput->m_isStopped

++             && m_videoOutput->m_currentVideoMode == QWasmVideoOutput::SurfaceCapture)

++         || m_videoOutput->isReady())

++         emscripten_request_animation_frame(frame, this);

++ }

++

++ QVideoFrameFormat::PixelFormat QWasmVideoFrameGrabber::fromJsPixelFormat(std::string_view videoFormat)

++ {

++     if (videoFormat == "I420")

++         return QVideoFrameFormat::Format_YUV420P;

++     // no equivalent pixel format

++     //   else if (videoFormat == "I420A") // AYUV ?

++     else if (videoFormat == "I422")

++         return QVideoFrameFormat::Format_YUV422P;

++     // no equivalent pixel format

++     //     else if (videoFormat == "I444")

++     else if (videoFormat == "NV12")

++         return QVideoFrameFormat::Format_NV12;

++     else if (videoFormat == "RGBA")

++         return QVideoFrameFormat::Format_RGBA8888;

++     else if (videoFormat == "RGBX")

++         return QVideoFrameFormat::Format_RGBX8888;

++     else if (videoFormat == "BGRA")

++         return QVideoFrameFormat::Format_BGRA8888;

++     else if (videoFormat == "BGRX")

++         return QVideoFrameFormat::Format_BGRX8888;

++

++     return QVideoFrameFormat::Format_Invalid;

++ }

++

++ QT_END_NAMESPACE

Revision control