Revision control
Copy as Markdown
// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qwasmvideooutput_p.h"
#include <QtMultimedia/qmediaplayer.h>
#include <QtMultimedia/qvideoframe.h>
#include <QtMultimedia/qvideosink.h>
#include <QtMultimedia/private/qplatformvideosink_p.h>
#include <QtGui/qguiapplication.h>
#include <QtGui/qopenglcontext.h>
#include <QtGui/qpa/qplatformwindow_p.h>
#include <QtGui/rhi/qrhi_platform.h>
#include <QtCore/qbuffer.h>
#include <QtCore/qdebug.h>
#include <QtCore/qfile.h>
#include <QtCore/qmimedatabase.h>
#include <QtCore/qpoint.h>
#include <QtCore/qpointer.h>
#include <QtCore/qrect.h>
#include <QtCore/qtimer.h>
#include <QtCore/qurl.h>
#include <QtCore/private/qstdweb_p.h>
#include <emscripten/bind.h>
#include <emscripten/val.h>
#include <GLES2/gl2.h>
QT_BEGIN_NAMESPACE
using namespace emscripten;
using namespace Qt::Literals;
Q_LOGGING_CATEGORY(qWasmMediaVideoOutput, "qt.multimedia.wasm.videooutput")
bool QWasmVideoOutput::isPlatformiOs()
{
emscripten::val platformObject = emscripten::val::global("navigator")["platform"];
if (platformObject.call<bool>("includes", emscripten::val("iPhone"))
|| platformObject.call<bool>("includes", emscripten::val("iPad")))
return true;
return false;
}
QWasmVideoOutput::QWasmVideoOutput(QObject *parent)
: QObject{ parent }, m_frameGrabber(this)
{
if (isPlatformiOs()) {
connect(this, &QWasmVideoOutput::orientationChanged, this,
&QWasmVideoOutput::applyIosRotation);
emscripten_set_orientationchange_callback(this, false, &QWasmVideoOutput::orientationchangeCallback);
}
}
QWasmVideoOutput::~QWasmVideoOutput()
{
if (m_mediaInputStream) {
if (m_streamStarted) {
m_streamStarted = false;
m_mediaInputStream->unregisterConsumer();
}
JsMediaInputStream::releaseInstance(m_cameraId);
}
}
void QWasmVideoOutput::setVideoSize(const QSize &newSize)
{
if (m_pendingVideoSize == newSize)
return;
m_pendingVideoSize = newSize;
updateVideoElementGeometry(QRect(0, 0, m_pendingVideoSize.width(), m_pendingVideoSize.height()));
}
void QWasmVideoOutput::setVideoMode(QWasmVideoOutput::WasmVideoMode mode)
{
m_currentVideoMode = mode;
}
// Calls play() on the video element and handles the returned promise.
//
// A play() that is interrupted by a new load request (e.g. a fresh srcObject or
// a load() call) rejects with AbortError. That rejection is benign, but if the
// promise is left unhandled the browser reports it as an uncaught rejection.
//
// A play() that is refused outright is not benign, the element stays unplayable
// and fires no further media events, so nothing else would take the player out
// of the state it is in. Report those instead of swallowing them.
void QWasmVideoOutput::playVideoElement()
{
emscripten::val promise = m_video.call<emscripten::val>("play");
// Older browsers return undefined from play() instead of a Promise. Only
// attach handlers when we actually got a Promise back.
if (promise.isUndefined() || promise.isNull() || promise["then"].isUndefined())
return;
QPointer<QWasmVideoOutput> videoOutput(this);
qstdweb::Promise::adoptPromise(promise, {
.catchFunc = [videoOutput](emscripten::val error) {
if (!videoOutput || videoOutput->m_isStopped)
return;
// The rejection value is not guaranteed to be a DOMException, so do
// not assume either property is there.
const auto stringProperty = [&error](const char *key) {
return error[key].isUndefined() || error[key].isNull()
? QString()
: QString::fromStdString(error[key].as<std::string>());
};
const QString errorName = stringProperty("name");
const QString errorMessage = stringProperty("message");
// The browsers word this differently and some do not mention play()
// at all, so keep the origin in the string the application sees.
const QString errorString =
"video.play(): "_L1 + (errorMessage.isEmpty() ? errorName : errorMessage);
// Not being allowed to play is a policy refusal, not an interruption.
// The element is not sitting at a resume point, it is unplayable
// until the user acts, so report it regardless of what the paused
// attribute happens to say.
if (errorName == "NotAllowedError"_L1) {
qCWarning(qWasmMediaVideoOutput) << "video.play() not allowed:" << errorMessage;
Q_EMIT videoOutput->readyChanged(false);
Q_EMIT videoOutput->stateChanged(QWasmMediaPlayer::Stopped);
Q_EMIT videoOutput->errorOccured(QMediaPlayer::AccessDeniedError, errorString);
return;
}
// Anything else is delivered a microtask late, and an AbortError from
// a new load request is routinely followed by a fresh play() that has
// already succeeded by now. Trust the element over the rejection.
if (!videoOutput->m_video["paused"].as<bool>()) {
qCDebug(qWasmMediaVideoOutput) << "video.play() rejected with" << errorName
<< "but playback resumed, ignoring";
return;
}
qCWarning(qWasmMediaVideoOutput) << "video.play() rejected:" << errorName
<< errorMessage;
Q_EMIT videoOutput->readyChanged(false);
Q_EMIT videoOutput->stateChanged(QWasmMediaPlayer::Stopped);
Q_EMIT videoOutput->errorOccured(QMediaPlayer::ResourceError, errorString);
}
});
}
void QWasmVideoOutput::start()
{
if (m_video.isUndefined() || m_video.isNull()
|| !m_wasmSink) {
// error
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error"));
return;
}
switch (m_currentVideoMode) {
case QWasmVideoOutput::VideoDisplay: {
emscripten::val sourceObj = m_video["src"];
if ((sourceObj.isUndefined() || sourceObj.isNull()) && !m_source.isEmpty()) {
m_video.set("src", m_source);
}
if (!isReady())
m_video.call<void>("load");
} break;
case QWasmVideoOutput::SurfaceCapture: {
playVideoElement();
Q_EMIT readyChanged(true);
} break;
case QWasmVideoOutput::Camera: {
{
emscripten::val document = emscripten::val::global("document");
if (m_video["parentNode"].isNull() || m_video["parentNode"].isUndefined())
document["body"].call<void>("appendChild", m_video);
}
if (!m_cameraIsReady) {
m_shouldBeStarted = true;
}
if (!m_connection)
m_connection = connect(m_mediaInputStream, &JsMediaInputStream::mediaVideoStreamReady, this,
[=]( ) {
emscripten::val newStream = m_mediaInputStream->getMediaStream();
// mediaVideoStreamReady may fire again for an already-attached
// stream (e.g. a restart on a still-active camera). Re-setting
// srcObject would interrupt the in-flight play() and reject its
// promise with AbortError, so skip it when nothing changed.
if (m_video["srcObject"].equals(newStream)) {
Q_EMIT readyChanged(true);
return;
}
m_video.set("srcObject", newStream);
emscripten::val stream = m_video["srcObject"];
if (stream.isNull() || stream.isUndefined()) { // camera device
qCDebug(qWasmMediaVideoOutput) << "srcObject ERROR";
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error"));
return;
} else {
emscripten::val videoTracks = stream.call<emscripten::val>("getVideoTracks");
if (videoTracks.isNull() || videoTracks.isUndefined()) {
qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "videoTracks is null";
Q_EMIT errorOccured(QMediaPlayer::ResourceError,
QStringLiteral("video surface error"));
return;
}
if (videoTracks["length"].as<int>() == 0) {
qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "videoTracks count is 0";
Q_EMIT errorOccured(QMediaPlayer::ResourceError,
QStringLiteral("video surface error"));
return;
}
emscripten::val videoSettings = videoTracks[0].call<emscripten::val>("getSettings");
if (!videoSettings.isNull() && !videoSettings.isUndefined()) {
const int width = videoSettings["width"].as<int>();
const int height = videoSettings["height"].as<int>();
updateVideoElementGeometry(QRect(0, 0, width, height));
if (!videoSettings["frameRate"].isUndefined())
m_streamFrameRate = videoSettings["frameRate"].as<double>();
}
}
playVideoElement();
Q_EMIT readyChanged(true);
});
m_mediaInputStream->setUseAudio(false);
m_shouldBeStarted = true;
m_mediaInputStream->setVideoConstraints(m_videoResolution, m_minFrameRate, m_maxFrameRate);
if (!m_streamStarted) {
m_streamStarted = true;
m_mediaInputStream->registerConsumer();
}
m_mediaInputStream->setStreamDevice(m_cameraId);
} break;
};
m_isStopped = false;
if (m_currentVideoMode != QWasmVideoOutput::Camera
&& m_currentVideoMode != QWasmVideoOutput::SurfaceCapture) {
playVideoElement();
}
}
void QWasmVideoOutput::stop()
{
if (m_isStopped)
return;
qCWarning(qWasmMediaVideoOutput) << Q_FUNC_INFO << "mode=" << m_currentVideoMode;
if (m_video.isUndefined() || m_video.isNull()) {
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error"));
return;
}
m_isStopped = true;
if (!m_toBePaused) {
if (m_currentVideoMode == QWasmVideoOutput::SurfaceCapture) {
emscripten::val stream = m_video["srcObject"];
if (!stream.isNull() && !stream.isUndefined()) {
emscripten::val tracks = stream.call<emscripten::val>("getTracks");
const int count = tracks["length"].as<int>();
for (int i = 0; i < count; ++i)
tracks[i].call<void>("stop");
}
} else if (m_mediaInputStream && m_streamStarted) {
// Only stop the shared MediaStream once the last consumer of this
// camera goes away; other displays of the same camera keep running.
m_streamStarted = false;
m_mediaInputStream->unregisterConsumer();
}
m_video.set("srcObject", emscripten::val::null());
disconnect(m_connection);
m_connection = {};
m_video.call<void>("remove");
} else {
m_video.call<void>("pause");
}
}
void QWasmVideoOutput::pause()
{
qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO;
if (m_video.isUndefined() || m_video.isNull()) {
// error
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error"));
return;
}
m_isStopped = false;
m_toBePaused = true;
m_video.call<void>("pause");
}
void QWasmVideoOutput::reset()
{
// flush pending frame
if (m_wasmSink)
m_wasmSink->platformVideoSink()->setVideoFrame(QVideoFrame());
m_source.clear();
m_video.set("currentTime", emscripten::val(0));
m_video.call<void>("load");
}
emscripten::val QWasmVideoOutput::surfaceElement()
{
return m_video;
}
void QWasmVideoOutput::setSurface(QVideoSink *surface)
{
if (!surface || surface == m_wasmSink) {
return;
}
m_wasmSink = surface;
}
bool QWasmVideoOutput::isReady() const
{
if (m_video.isUndefined() || m_video.isNull()) {
// error
return false;
}
return m_currentMediaStatus == MediaStatus::LoadedMedia;
}
void QWasmVideoOutput::setSource(const QUrl &url)
{
qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << url;
m_source = url.toString();
if (m_video.isUndefined() || m_video.isNull()) {
return;
}
if (url.isEmpty()) {
stop();
return;
}
if (url.isLocalFile()) {
QFile localFile(url.toLocalFile());
if (localFile.open(QIODevice::ReadOnly)) {
setSource(&localFile);
} else {
qWarning() << "Failed to open file";
}
return;
}
updateVideoElementSource(m_source);
}
void QWasmVideoOutput::updateVideoElementSource(const QString &src)
{
m_video.set("src", src.toStdString());
m_video.call<void>("load");
}
void QWasmVideoOutput::addCameraSourceElement(const std::string &id)
{
m_cameraIsReady = false;
if (m_mediaInputStream)
JsMediaInputStream::releaseInstance(m_cameraId);
m_mediaInputStream = JsMediaInputStream::instance(id);
m_mediaInputStream->setUseAudio(m_hasAudio);
m_mediaInputStream->setUseVideo(true);
connect(m_mediaInputStream, &JsMediaInputStream::mediaVideoStreamReady, this,
[this]() {
qCDebug(qWasmMediaVideoOutput) << "mediaVideoStreamReady" << m_shouldBeStarted;
m_cameraIsReady = true;
if (m_shouldBeStarted) {
start();
m_shouldBeStarted = false;
}
});
m_cameraId = id;
}
void QWasmVideoOutput::setVideoConstraints(QSize resolution, float minFrameRate, float maxFrameRate)
{
m_videoResolution = resolution;
m_minFrameRate = minFrameRate;
m_maxFrameRate = maxFrameRate;
}
void QWasmVideoOutput::setSource(QIODevice *stream)
{
if (stream->bytesAvailable() == 0) {
qWarning() << "data not available";
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("data not available"));
return;
}
if (m_video.isUndefined() || m_video.isNull()) {
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error"));
return;
}
QMimeDatabase db;
QMimeType mime = db.mimeTypeForData(stream);
QByteArray buffer = stream->readAll();
qstdweb::Blob contentBlob = qstdweb::Blob::copyFrom(buffer.data(), buffer.size(), mime.name().toStdString());
emscripten::val window = qstdweb::window();
if (window["safari"].isUndefined()) {
emscripten::val contentUrl = window["URL"].call<emscripten::val>("createObjectURL", contentBlob.val());
m_video.set("src", contentUrl);
m_source = QString::fromStdString(contentUrl.as<std::string>());
} else {
// only Safari currently supports Blob with srcObject
m_video.set("srcObject", contentBlob.val());
}
}
void QWasmVideoOutput::setVolume(qreal volume)
{ // between 0 - 1
volume = qBound(qreal(0.0), volume, qreal(1.0));
m_video.set("volume", volume);
}
void QWasmVideoOutput::setMuted(bool muted)
{
if (m_video.isUndefined() || m_video.isNull()) {
// error
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error"));
return;
}
m_video.set("muted", muted);
}
qint64 QWasmVideoOutput::getCurrentPosition()
{
return (!m_video.isUndefined() || !m_video.isNull())
? (m_video["currentTime"].as<double>() * 1000)
: 0;
}
void QWasmVideoOutput::seekTo(qint64 positionMSecs)
{
if (isVideoSeekable()) {
float positionToSetInSeconds = float(positionMSecs) / 1000;
emscripten::val seekableTimeRange = m_video["seekable"];
if (!seekableTimeRange.isNull() || !seekableTimeRange.isUndefined()) {
// range user can seek
if (seekableTimeRange["length"].as<int>() < 1)
return;
if (positionToSetInSeconds
>= seekableTimeRange.call<emscripten::val>("start", 0).as<double>()
&& positionToSetInSeconds
<= seekableTimeRange.call<emscripten::val>("end", 0).as<double>()) {
m_requestedPosition = positionToSetInSeconds;
m_video.set("currentTime", m_requestedPosition);
}
}
}
qCDebug(qWasmMediaVideoOutput) << "m_requestedPosition" << m_requestedPosition;
}
bool QWasmVideoOutput::isVideoSeekable()
{
if (m_video.isUndefined() || m_video.isNull()) {
// error
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error"));
return false;
}
emscripten::val seekableTimeRange = m_video["seekable"];
if (seekableTimeRange["length"].as<int>() < 1)
return false;
if (!seekableTimeRange.isNull() || !seekableTimeRange.isUndefined()) {
bool isit = !QtPrivate::fuzzyCompare(
seekableTimeRange.call<emscripten::val>("start", 0).as<double>(),
seekableTimeRange.call<emscripten::val>("end", 0).as<double>());
return isit;
}
return false;
}
void QWasmVideoOutput::createVideoElement(const std::string &id)
{
qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << this << id;
// Create <video> element and add it to the page body
emscripten::val document = emscripten::val::global("document");
emscripten::val body = document["body"];
// remove any previously created video element for this output
if (!m_video.isUndefined() && !m_video.isNull())
m_video.call<void>("remove");
m_videoSurfaceId = id;
m_video = document.call<emscripten::val>("createElement", std::string("video"));
m_video.set("id", m_videoSurfaceId.c_str());
m_video.call<void>("setAttribute", std::string("class"),
(m_currentVideoMode == QWasmVideoOutput::Camera ? std::string("Camera")
: std::string("Video")));
m_video.set("preload", "metadata");
// Uncaught DOMException: Failed to execute 'getImageData' on
// 'OffscreenCanvasRenderingContext2D': The canvas has been tainted by
// cross-origin data.
// TODO figure out somehow to let user choose between these
std::string originString = "anonymous"; // requires server Access-Control-Allow-Origin *
// std::string originString = "use-credentials"; // must not
// Access-Control-Allow-Origin *
m_video.call<void>("setAttribute", std::string("crossorigin"), originString);
body.call<void>("appendChild", m_video);
// Create/add video source
document.call<emscripten::val>("createElement",
std::string("source")).set("src", m_source.toStdString());
// Set position:absolute, which makes it possible to position the video
// element using x,y. coordinates, relative to its parent (the page's <body>
// element)
emscripten::val style = m_video["style"];
style.set("position", "absolute");
style.set("display", "none"); // hide
if (!m_source.isEmpty())
updateVideoElementSource(m_source);
}
void QWasmVideoOutput::removeCurrentVideoElement()
{
if (!m_video.isUndefined() && !m_video.isNull())
m_video.call<void>("remove");
}
void QWasmVideoOutput::updateVideoElementGeometry(const QRect &windowGeometry)
{
QRect videoElementRect(windowGeometry.topLeft(), windowGeometry.size());
emscripten::val style = m_video["style"];
style.set("left", QStringLiteral("%1px").arg(videoElementRect.left()).toStdString());
style.set("top", QStringLiteral("%1px").arg(videoElementRect.top()).toStdString());
m_video.set("width", videoElementRect.width());
m_video.set("height", videoElementRect.height());
style.set("z-index", "999");
}
qint64 QWasmVideoOutput::getDuration()
{
// qt duration is in ms
// js is sec
if (m_video.isUndefined() || m_video.isNull())
return 0;
return m_video["duration"].as<double>() * 1000;
}
void QWasmVideoOutput::setPlaybackRate(qreal rate)
{
m_video.set("playbackRate", emscripten::val(rate));
}
qreal QWasmVideoOutput::playbackRate()
{
return (m_video.isUndefined() || m_video.isNull()) ? 0 : m_video["playbackRate"].as<float>();
}
emscripten::val QWasmVideoOutput::getDeviceCapabilities()
{
emscripten::val stream = m_video["srcObject"];
if ((!stream.isNull() && !stream.isUndefined()) && stream["active"].as<bool>()) {
emscripten::val tracks = stream.call<emscripten::val>("getVideoTracks");
if (!tracks.isUndefined()) {
if (tracks["length"].as<int>() == 0)
return emscripten::val::undefined();
emscripten::val track = tracks[0];
if (!track.isUndefined()) {
emscripten::val trackCaps = emscripten::val::undefined();
if (!track["getCapabilities"].isUndefined())
trackCaps = track.call<emscripten::val>("getCapabilities");
else // firefox does not support getCapabilities
trackCaps = track.call<emscripten::val>("getSettings");
if (!trackCaps.isUndefined())
return trackCaps;
}
}
} else {
// camera not started track capabilities not available
Q_EMIT errorOccured(QMediaPlayer::ResourceError, QStringLiteral("capabilities not available"));
}
return emscripten::val::undefined();
}
bool QWasmVideoOutput::setDeviceSetting(const std::string &key, emscripten::val value)
{
emscripten::val stream = m_video["srcObject"];
if (stream.isNull() || stream.isUndefined()
|| stream["getVideoTracks"].isUndefined())
return false;
emscripten::val tracks = stream.call<emscripten::val>("getVideoTracks");
if (!tracks.isNull() || !tracks.isUndefined()) {
if (tracks["length"].as<int>() == 0)
return false;
emscripten::val track = tracks[0];
emscripten::val contraint = emscripten::val::object();
contraint.set(std::move(key), value);
track.call<emscripten::val>("applyConstraints", contraint);
return true;
}
return false;
}
QT_END_NAMESPACE
#include "moc_qwasmvideooutput_p.cpp"