Revision control
++ // Copyright (C) 2025 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ #include "qwasmjs_p.h"
++ #include <qaudiodevice.h>
++ #include <qcameradevice.h>
++ #include <QTimer>
++
++ QT_BEGIN_NAMESPACE
++
++
++ namespace {
++ struct MediaInputStreamEntry
++ {
++ JsMediaInputStream *stream = nullptr;
++ int referenceCount = 0;
++ };
++ using MediaInputStreamMap = QMap<std::string, MediaInputStreamEntry>;
++ }
++
++ Q_GLOBAL_STATIC(MediaInputStreamMap, s_wasmMediaInputStreams);
++
++ JsMediaRecorder::JsMediaRecorder() = default;
++
++ bool JsMediaRecorder::open(QIODevice::OpenMode mode)
++ {
++ if (mode.testFlag(QIODevice::WriteOnly))
++ return false;
++ return QIODevice::open(mode);
++ }
++
++ bool JsMediaRecorder::isSequential() const
++ {
++ return false;
++ }
++
++ qint64 JsMediaRecorder::size() const
++ {
++ return m_buffer.size();
++ }
++
++ bool JsMediaRecorder::seek(qint64 pos)
++ {
++ if (pos >= size())
++ return false;
++ return QIODevice::seek(pos);
++ }
++
++ qint64 JsMediaRecorder::readData(char *data, qint64 maxSize)
++ {
++ qint64 bytesToRead = qMin(maxSize, (qint64)m_buffer.size());
++ memcpy(data, m_buffer.constData(), bytesToRead);
++ m_buffer = m_buffer.right(m_buffer.size() - bytesToRead);
++ return bytesToRead;
++ }
++
++ qint64 JsMediaRecorder::writeData(const char *, qint64)
++ {
++ Q_UNREACHABLE_RETURN(0);
++ }
++
++ void JsMediaRecorder::audioDataAvailable(emscripten::val aBlob, double timeCodeDifference)
++ {
++ Q_UNUSED(timeCodeDifference)
++ if (aBlob.isUndefined() || aBlob.isNull()) {
++ qWarning() << "blob is null";
++ return;
++ }
++
++ auto fileReader = std::make_shared<qstdweb::FileReader>();
++
++ fileReader->onError([=](emscripten::val theError) {
++ Q_EMIT streamError(QMediaRecorder::ResourceError,
++ QString::fromStdString(theError["message"].as<std::string>()));
++ });
++
++ fileReader->onAbort([=](emscripten::val) {
++ Q_EMIT streamError(QMediaRecorder::ResourceError, QStringLiteral("File read aborted"));
++ });
++
++ fileReader->onLoad([=](emscripten::val) {
++ if (fileReader->val().isNull() || fileReader->val().isUndefined())
++ return;
++ qstdweb::ArrayBuffer result = fileReader->result();
++ if (result.val().isNull() || result.val().isUndefined())
++ return;
++
++ m_buffer.append(qstdweb::Uint8Array(result).copyToQByteArray());
++ Q_EMIT readyRead();
++ });
++
++ fileReader->readAsArrayBuffer(qstdweb::Blob(aBlob));
++ }
++
++ void JsMediaRecorder::setTrackContraints(QMediaEncoderSettings &settings, emscripten::val stream)
++ {
++ if (stream.isUndefined() || stream.isNull()) {
++ qWarning()<< "could not find MediaStream";
++ return;
++ }
++
++ emscripten::val navigator = emscripten::val::global("navigator");
++ emscripten::val mediaDevices = navigator["mediaDevices"];
++
++ // check which ones are supported
++ emscripten::val allConstraints = mediaDevices.call<emscripten::val>("getSupportedConstraints");
++ // browsers only support some settings
++
++ emscripten::val videoParams = emscripten::val::object();
++ emscripten::val constraints = emscripten::val::object();
++ videoParams.set("resizeMode",std::string("crop-and-scale"));
++
++ if (m_needsCamera) {
++ if (settings.videoFrameRate() > 0)
++ videoParams.set("frameRate", emscripten::val(settings.videoFrameRate()));
++ if (settings.videoResolution().height() > 0)
++ videoParams.set("height",
++ emscripten::val(settings.videoResolution().height())); // viewportHeight?
++ if (settings.videoResolution().width() > 0)
++ videoParams.set("width", emscripten::val(settings.videoResolution().width()));
++
++ constraints.set("video", videoParams); // only video here
++ }
++
++ emscripten::val audioParams = emscripten::val::object();
++ if (settings.audioSampleRate() > 0)
++ audioParams.set("sampleRate", emscripten::val(settings.audioSampleRate())); // may not work
++ if (settings.audioBitRate() > 0)
++ audioParams.set("sampleSize", emscripten::val(settings.audioBitRate())); // may not work
++ if (settings.audioChannelCount() > 0)
++ audioParams.set("channelCount", emscripten::val(settings.audioChannelCount()));
++
++ constraints.set("audio", audioParams); // only audio here
++
++ if (m_needsCamera && stream["active"].as<bool>()) {
++ emscripten::val videoTracks = emscripten::val::undefined();
++ videoTracks = stream.call<emscripten::val>("getVideoTracks");
++ if (videoTracks.isNull() || videoTracks.isUndefined()) {
++ qWarning() << "no video tracks";
++ return;
++ }
++ if (videoTracks["length"].as<int>() > 0) {
++ // try to apply the video options, async
++ qstdweb::Promise::make(videoTracks[0],
++ QStringLiteral("applyConstraints"), {
++ .thenFunc =
++ [this]([[maybe_unused]] emscripten::val result) {
++ startStreaming();
++ },
++ .catchFunc =
++ [this](emscripten::val theError) {
++ qWarning()
++ << theError["code"].as<int>()
++ << theError["message"].as<std::string>();
++ Q_EMIT streamError(QMediaRecorder::ResourceError,
++ QString::fromStdString(theError["message"].as<std::string>()));
++ },
++ .finallyFunc = []() {},
++ },
++ constraints);
++ }
++ }
++ }
++
++ void JsMediaRecorder::pauseStream()
++ {
++ if (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull()) {
++ qWarning() << "could not find MediaRecorder";
++ return;
++ }
++ m_mediaRecorder.call<void>("pause");
++ }
++
++ void JsMediaRecorder::resumeStream()
++ {
++ if (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull()) {
++ qWarning() << "could not find MediaRecorder";
++ return;
++ }
++
++ m_mediaRecorder.call<void>("resume");
++ }
++
++ void JsMediaRecorder::stopStream()
++ {
++ if (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull()) {
++ qWarning()<< "could not find MediaRecorder";
++ return;
++ }
++ if (m_mediaRecorder["state"].as<std::string>() == "recording")
++ m_mediaRecorder.call<void>("stop");
++
++ }
++
++ void JsMediaRecorder::startStreaming()
++ {
++ if (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull()) {
++ qWarning() << "could not find MediaStream";
++ return;
++ }
++
++ constexpr int sliceSizeInMs = 256;
++ // AudioWorklets uses 128 by default
++ m_mediaRecorder.call<void>("start", emscripten::val(sliceSizeInMs));
++ }
++
++ void JsMediaRecorder::setStream(emscripten::val stream)
++ {
++ emscripten::val emMediaSettings = emscripten::val::object();
++ QMediaFormat::VideoCodec videoCodec = m_mediaSettings.videoCodec();
++ QMediaFormat::AudioCodec audioCodec = m_mediaSettings.audioCodec();
++ QMediaFormat::FileFormat fileFormat = m_mediaSettings.fileFormat();
++
++ // mime and codecs
++ QString mimeCodec;
++ if (!m_mediaSettings.mimeType().name().isEmpty()) {
++ mimeCodec = m_mediaSettings.mimeType().name();
++
++ if (videoCodec != QMediaFormat::VideoCodec::Unspecified)
++ mimeCodec += QStringLiteral(": codecs=");
++
++ if (audioCodec != QMediaFormat::AudioCodec::Unspecified) {
++ // TODO
++ }
++
++ if (fileFormat != QMediaFormat::UnspecifiedFormat)
++ mimeCodec += QMediaFormat::fileFormatName(m_mediaSettings.fileFormat());
++
++ emMediaSettings.set("mimeType", mimeCodec.toStdString());
++ }
++
++ if (m_mediaSettings.audioBitRate() > 0)
++ emMediaSettings.set("audioBitsPerSecond", emscripten::val(m_mediaSettings.audioBitRate()));
++
++ if (m_mediaSettings.videoBitRate() > 0)
++ emMediaSettings.set("videoBitsPerSecond", emscripten::val(m_mediaSettings.videoBitRate()));
++
++ // create the MediaRecorder, and set up data callback
++ m_mediaRecorder = emscripten::val::global("MediaRecorder").new_(stream, emMediaSettings);
++
++ if (m_mediaRecorder.isNull() || m_mediaRecorder.isUndefined()) {
++ qWarning() << "MediaRecorder could not be found";
++ return;
++ }
++ m_mediaRecorder.set("data-mediarecordercontext",
++ emscripten::val(quintptr(reinterpret_cast<void *>(this))));
++
++ if (!m_mediaStreamDataAvailable.isNull()) {
++ m_mediaStreamDataAvailable.reset();
++ m_mediaStreamStopped.reset();
++ m_mediaStreamError.reset();
++ m_mediaStreamStart.reset();
++ m_mediaStreamPause.reset();
++ m_mediaStreamResume.reset();
++ }
++
++ // dataavailable
++ auto callback = [](emscripten::val blob) {
++ if (blob.isUndefined() || blob.isNull()) {
++ qWarning() << "blob is null";
++ return;
++ }
++ if (blob["target"].isUndefined() || blob["target"].isNull())
++ return;
++ if (blob["data"].isUndefined() || blob["data"].isNull())
++ return;
++ if (blob["target"]["data-mediarecordercontext"].isUndefined()
++ || blob["target"]["data-mediarecordercontext"].isNull())
++ return;
++
++ JsMediaRecorder *recorder = reinterpret_cast<JsMediaRecorder *>(
++ blob["target"]["data-mediarecordercontext"].as<quintptr>());
++
++ if (recorder) {
++ const double timeCode =
++ blob.hasOwnProperty("timecode") ? blob["timecode"].as<double>() : 0;
++ recorder->audioDataAvailable(blob["data"], timeCode);
++ }
++ };
++
++ m_mediaStreamDataAvailable.reset(
++ new qstdweb::EventCallback(m_mediaRecorder, "dataavailable", callback));
++
++ // stopped
++ auto stoppedCallback = [this](emscripten::val event) {
++ if (event.isUndefined() || event.isNull()) {
++ qWarning() << "event is null";
++ return;
++ }
++ m_currentState = QMediaRecorder::StoppedState;
++ JsMediaRecorder *recorder = reinterpret_cast<JsMediaRecorder *>(
++ event["target"]["data-mediarecordercontext"].as<quintptr>());
++ Q_EMIT recorder->stopped();
++ };
++
++ m_mediaStreamStopped.reset(
++ new qstdweb::EventCallback(m_mediaRecorder, "stop", stoppedCallback));
++
++ // error
++ auto errorCallback = [this](emscripten::val theError) {
++ if (theError.isUndefined() || theError.isNull()) {
++ qWarning() << "error is null";
++ return;
++ }
++
++ Q_EMIT streamError(QMediaRecorder::ResourceError,
++ QString::fromStdString(theError["message"].as<std::string>()));
++ };
++
++ m_mediaStreamError.reset(new qstdweb::EventCallback(m_mediaRecorder, "error", errorCallback));
++
++ // start
++ auto startCallback = [this](emscripten::val event) {
++ if (event.isUndefined() || event.isNull()) {
++ qWarning() << "event is null";
++ return;
++ }
++
++ JsMediaRecorder *recorder = reinterpret_cast<JsMediaRecorder *>(
++ event["target"]["data-mediarecordercontext"].as<quintptr>());
++ m_currentState = QMediaRecorder::RecordingState;
++ Q_EMIT recorder->started();
++ };
++
++ m_mediaStreamStart.reset(new qstdweb::EventCallback(m_mediaRecorder, "start", startCallback));
++
++ // pause
++ auto pauseCallback = [this](emscripten::val event) {
++ if (event.isUndefined() || event.isNull()) {
++ qWarning() << "event is null";
++ return;
++ }
++
++ JsMediaRecorder *recorder = reinterpret_cast<JsMediaRecorder *>(
++ event["target"]["data-mediarecordercontext"].as<quintptr>());
++ m_currentState = QMediaRecorder::PausedState;
++ Q_EMIT recorder->paused();
++ };
++
++ m_mediaStreamPause.reset(new qstdweb::EventCallback(m_mediaRecorder, "pause", pauseCallback));
++
++ // resume
++ auto resumeCallback = [this](emscripten::val event) {
++ if (event.isUndefined() || event.isNull()) {
++ qWarning() << "event is null";
++ return;
++ }
++ m_currentState = QMediaRecorder::RecordingState;
++
++ JsMediaRecorder *recorder = reinterpret_cast<JsMediaRecorder *>(
++ event["target"]["data-mediarecordercontext"].as<quintptr>());
++ Q_EMIT recorder->resumed();
++ };
++
++ m_mediaStreamResume.reset(
++ new qstdweb::EventCallback(m_mediaRecorder, "resume", resumeCallback));
++ }
++
++ qint64 JsMediaRecorder::bytesAvailable() const
++ {
++ return m_buffer.size();
++ }
++
++
++ JsMediaInputStream::JsMediaInputStream(QObject *parent)
++ : QObject{parent}
++ {
++ }
++
++ JsMediaInputStream::~JsMediaInputStream() = default;
++
++ JsMediaInputStream *JsMediaInputStream::instance(const std::string &deviceId)
++ {
++ MediaInputStreamEntry &entry = (*s_wasmMediaInputStreams())[deviceId];
++ if (!entry.stream)
++ entry.stream = new JsMediaInputStream;
++ ++entry.referenceCount;
++ return entry.stream;
++ }
++
++ void JsMediaInputStream::releaseInstance(const std::string &deviceId)
++ {
++ auto it = s_wasmMediaInputStreams()->find(deviceId);
++ if (it == s_wasmMediaInputStreams()->end())
++ return;
++ if (--it->referenceCount <= 0) {
++ delete it->stream;
++ s_wasmMediaInputStreams()->erase(it);
++ }
++ }
++
++ void JsMediaInputStream::unregisterConsumer()
++ {
++ if (--m_consumerCount <= 0) {
++ m_consumerCount = 0;
++ stopMediaStream(m_mediaStream);
++ }
++ }
++
++ void JsMediaInputStream::replaceAudioStreamDevice(const std::string &audioDeviceId)
++ {
++ for (auto it = s_wasmMediaInputStreams()->begin(); it != s_wasmMediaInputStreams()->end(); ++it) {
++ JsMediaInputStream *stream = it->stream;
++ if (stream && stream->m_needsAudio)
++ stream->setAudioStreamDevice(audioDeviceId);
++ }
++ }
++
++ void JsMediaInputStream::setVideoConstraints(QSize resolution, float minFrameRate, float maxFrameRate)
++ {
++ m_videoResolution = resolution;
++ m_minFrameRate = minFrameRate;
++ m_maxFrameRate = maxFrameRate;
++ }
++
++ void JsMediaInputStream::setAudioStreamDevice(const std::string &id)
++ {
++ if (!m_mediaStream.isUndefined() && !m_mediaStream.isNull()) {
++ if (!m_mediaStream.isNull() && !m_mediaStream.isUndefined()
++ && !m_mediaStream["getTracks"].isUndefined() && m_mediaStream["active"].as<bool>()) {
++ m_needsVideo = false;
++ m_needsAudio = true;
++ replaceMediaTrack(id);
++ }
++ }
++ }
++
++ void JsMediaInputStream::replaceMediaTrack(const std::string &id)
++ {
++ qstdweb::PromiseCallbacks getUserMediaCallback{
++ // default
++ .thenFunc =
++ [this, id](emscripten::val newStream) {
++
++ std::string getTracksCommand;
++ if (m_needsAudio)
++ getTracksCommand = "getAudioTracks";
++ else
++ getTracksCommand = "getVideoTracks";
++
++ emscripten::val currentTracks = m_mediaStream.call<emscripten::val>(getTracksCommand.c_str());
++
++ if (!currentTracks.isUndefined() && currentTracks["length"].as<int>() > 0) {
++ emscripten::val currentTrackForType = currentTracks[0];
++ emscripten::val settings = currentTrackForType.call<emscripten::val>("getSettings");
++
++ if (!settings.isNull() && !settings.isUndefined()) {
++ if (settings["deviceId"].as<std::string>() != id) {
++ m_mediaStream.call<void>("removeTrack", currentTrackForType);
++ currentTrackForType.call<void>("stop");
++
++ emscripten::val newTracks = newStream.call<emscripten::val>(getTracksCommand.c_str());
++
++ m_mediaStream.call<void>("addTrack", newTracks[0]);
++ newStream.call<void>("removeTrack", newTracks[0]);
++
++ // stopMediaStream(stream); stopping this stream causes the track to stop :(
++ if (m_needsAudio)
++ Q_EMIT mediaAudioStreamReady();
++ else
++ Q_EMIT mediaVideoStreamReady();
++ }
++ }
++ } else { // we still need to add this track
++ qWarning() << " we still need to add this track";
++ }
++ },
++ .catchFunc =
++ [](emscripten::val error) {
++ qWarning()
++ << "replaceTrack getUserMedia failed."
++ << error["name"].as<std::string>()
++ << error["message"].as<std::string>();
++ },
++ .finallyFunc = nullptr
++ };
++
++ emscripten::val mediaDevices = emscripten::val::global("navigator")["mediaDevices"];
++ qstdweb::Promise::make(mediaDevices, QStringLiteral("getUserMedia"),
++ std::move(getUserMediaCallback), setDeviceConstraints(id));
++ }
++
++ emscripten::val JsMediaInputStream::setDeviceConstraints(const std::string &id)
++ {
++ std::string deviceIdString = id;
++ if (deviceIdString.find("System") != std::string::npos)
++ deviceIdString.clear(); // no id is default/any device
++
++ emscripten::val constraints = emscripten::val::object();
++ if (m_needsAudio) {
++ emscripten::val audioConstraints = emscripten::val::object();
++ audioConstraints.set("audio", m_needsAudio);
++ if (!deviceIdString.empty()) {
++ emscripten::val exactDeviceId = emscripten::val::object();
++ exactDeviceId.set("exact", deviceIdString);
++ audioConstraints.set("deviceId", exactDeviceId);
++ }
++ constraints.set("audio", audioConstraints);
++ } else {
++ constraints.set("audio", false);
++ }
++
++ if (m_needsVideo) {
++ emscripten::val videoContraints = emscripten::val::object();
++ if (!deviceIdString.empty()) {
++ emscripten::val exactDeviceId = emscripten::val::object();
++ exactDeviceId.set("exact", deviceIdString);
++ videoContraints.set("deviceId", exactDeviceId);
++ }
++ videoContraints.set("resizeMode", std::string("crop-and-scale"));
++ if (m_videoResolution.isValid()) {
++ videoContraints.set("width", emscripten::val(m_videoResolution.width()));
++ videoContraints.set("height", emscripten::val(m_videoResolution.height()));
++ }
++ if (m_minFrameRate > 0 || m_maxFrameRate > 0) {
++ emscripten::val frameRateConstraint = emscripten::val::object();
++ if (m_minFrameRate > 0)
++ frameRateConstraint.set("min", emscripten::val(m_minFrameRate));
++ if (m_maxFrameRate > 0)
++ frameRateConstraint.set("max", emscripten::val(m_maxFrameRate));
++ videoContraints.set("frameRate", frameRateConstraint);
++ }
++ constraints.set("video", videoContraints);
++ }
++ return constraints;
++ }
++
++ void JsMediaInputStream::setStreamDevice(const std::string &id)
++ {
++ emscripten::val navigator = emscripten::val::global("navigator");
++ emscripten::val mediaDevices = navigator["mediaDevices"];
++
++ if (mediaDevices.isNull() || mediaDevices.isUndefined()) {
++ qWarning() << "No media devices found";
++ return;
++ }
++
++ // decide if we need to replace a track here
++
++ if (!m_mediaStream.isNull() && !m_mediaStream.isUndefined())
++ m_active = m_mediaStream["active"].as<bool>();
++
++ if (m_active) {
++ // The stream is already running and may be shared with another consumer
++ // (for example a second display of the same camera). A MediaStream can
++ // drive multiple video elements at once, so just notify the new consumer
++ // to attach to the existing stream. Deferred to avoid re-entering
++ // start() synchronously.
++ QTimer::singleShot(0, this, [this]() {
++ if (m_needsVideo)
++ Q_EMIT mediaVideoStreamReady();
++ if (m_needsAudio)
++ Q_EMIT mediaAudioStreamReady();
++ });
++ return;
++ }
++
++ qstdweb::PromiseCallbacks getUserMediaCallback{
++ // default
++ .thenFunc =
++ [this](emscripten::val stream) {
++ setupMediaStream(stream);
++ },
++ .catchFunc =
++ [](emscripten::val error) {
++ qWarning()
++ << "setStreamDevice getUserMedia fail"
++ << error["name"].as<std::string>()
++ << error["message"].as<std::string>();
++ },
++ .finallyFunc = nullptr
++ };
++
++ // this prompts user for permissions
++ qstdweb::Promise::make(mediaDevices, QStringLiteral("getUserMedia"),
++ std::move(getUserMediaCallback), setDeviceConstraints(id));
++ }
++
++ void JsMediaInputStream::setupMediaStream(emscripten::val mStream)
++ {
++ m_mediaStream = mStream;
++ m_active = mStream["active"].as<bool>();
++
++ auto activeStreamCallback = [=](emscripten::val) {
++ m_active = true;
++ Q_EMIT activated(m_active);
++ };
++ m_activeStreamEvent.reset(new qstdweb::EventCallback(m_mediaStream, "active", activeStreamCallback));
++
++ auto inactiveStreamCallback = [=](emscripten::val) {
++ m_active = false;
++ Q_EMIT activated(m_active);
++ };
++ m_inactiveStreamEvent.reset(new qstdweb::EventCallback(m_mediaStream, "inactive", inactiveStreamCallback));
++
++ if (m_needsAudio)
++ Q_EMIT mediaAudioStreamReady();
++ if (m_needsVideo)
++ Q_EMIT mediaVideoStreamReady();
++ }
++
++ void JsMediaInputStream::stopMediaStream(emscripten::val mediaStream)
++ {
++ if (!mediaStream.isNull() && !mediaStream.isUndefined() && !mediaStream["getTracks"].isUndefined()) {
++ emscripten::val tracks = mediaStream.call<emscripten::val>("getTracks");
++ if (!tracks.isUndefined() && tracks["length"].as<int>() > 0) {
++ for (int i = 0; i < tracks["length"].as<int>(); i++) {
++ tracks[i].call<void>("stop");
++ }
++ }
++ }
++ mediaStream = emscripten::val::undefined();
++ m_active = false;
++ }
++
++ QT_END_NAMESPACE