Revision control
++ // Copyright (C) 2016 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ #include "qgstreamerimagecapture_p.h"
++
++ #include <QtMultimedia/qvideoframeformat.h>
++ #include <QtMultimedia/private/qmediastoragelocation_p.h>
++ #include <QtMultimedia/private/qplatformcamera_p.h>
++ #include <QtMultimedia/private/qplatformimagecapture_p.h>
++ #include <QtMultimedia/private/qvideoframe_p.h>
++ #include <QtGui/qguiapplication.h>
++ #include <QtCore/qdebug.h>
++ #include <QtCore/qdir.h>
++ #include <QtCore/qstandardpaths.h>
++ #include <QtCore/qcoreapplication.h>
++ #include <QtCore/qloggingcategory.h>
++
++ #include <common/qgstreamermetadata_p.h>
++ #include <common/qgstvideobuffer_p.h>
++ #include <common/qgstutils_p.h>
++
++ #include <utility>
++
++ QT_BEGIN_NAMESPACE
++
++ namespace {
++
++ Q_STATIC_LOGGING_CATEGORY(qLcImageCaptureGst, "qt.multimedia.imageCapture")
++
++ struct ThreadPoolSingleton
++ {
++ QObject m_context;
++ QMutex m_poolMutex;
++ QThreadPool *m_instance{};
++ bool m_appUnderDestruction = false;
++
++ QThreadPool *get(const QMutexLocker<QMutex> &)
++ {
++ if (m_instance)
++ return m_instance;
++ if (m_appUnderDestruction || !qApp)
++ return nullptr;
++
++ using namespace std::chrono;
++
++ m_instance = new QThreadPool;
++ m_instance->setMaxThreadCount(1); // 1 thread;
++ static constexpr auto expiryTimeout = minutes(5);
++ m_instance->setExpiryTimeout(round<milliseconds>(expiryTimeout).count());
++
++ QObject::connect(qApp, &QCoreApplication::aboutToQuit, &m_context, [&] {
++ // we need to make sure that thread-local QRhi is destroyed before the application to
++ // prevent QTBUG-124189
++ QMutexLocker guard(&m_poolMutex);
++ delete m_instance;
++ m_instance = {};
++ m_appUnderDestruction = true;
++ });
++
++ QObject::connect(qApp, &QCoreApplication::destroyed, &m_context, [&] {
++ m_appUnderDestruction = false;
++ });
++ return m_instance;
++ }
++
++ template <typename Functor>
++ QFuture<void> run(Functor &&f)
++ {
++ QMutexLocker guard(&m_poolMutex);
++ QThreadPool *pool = get(guard);
++ if (!pool)
++ return QFuture<void>{};
++
++ return QtConcurrent::run(pool, std::forward<Functor>(f));
++ }
++ };
++
++ ThreadPoolSingleton s_threadPoolSingleton;
++
++ }; // namespace
++
++ template <typename Functor>
++ void QGstreamerImageCapture::invokeDeferred(Functor &&fn)
++ {
++ QMetaObject::invokeMethod(this, std::forward<decltype(fn)>(fn), Qt::QueuedConnection);
++ }
++
++ template <typename Functor>
++ void QGstreamerImageCapture::runInThreadPool(Functor fn)
++ {
++ int futureId = m_futureIDAllocator.fetch_add(1, std::memory_order_relaxed);
++
++ QFuture<void> future = QtConcurrent::run([this, futureId, fn = std::move(fn)]() mutable {
++ auto cleanup = qScopeGuard([&] {
++ QMutexLocker guard(&m_pendingFuturesMutex);
++ m_pendingFutures.erase(futureId);
++ });
++ fn();
++ });
++
++ if (!future.isValid()) // during qApplication shutdown the threadpool becomes unusable
++ return;
++
++ QMutexLocker guard(&m_pendingFuturesMutex);
++ m_pendingFutures.emplace(futureId, std::move(future));
++ }
++
++ q23::expected<QPlatformImageCapture *, QString> QGstreamerImageCapture::create(QImageCapture *parent)
++ {
++ static const auto error = qGstErrorMessageIfElementsNotAvailable(
++ "queue", "capsfilter", "videoconvert", "jpegenc", "jifmux", "fakesink");
++ if (error)
++ return q23::unexpected{ *error };
++
++ return new QGstreamerImageCapture(parent);
++ }
++
++ QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent)
++ : QPlatformImageCapture(parent),
++ QGstreamerBufferProbe(ProbeBuffers),
++ bin{
++ QGstBin::create("imageCaptureBin"),
++ },
++ queue{
++ QGstElement::createFromFactory("queue", "imageCaptureQueue"),
++ },
++ filter{
++ QGstElement::createFromFactory("capsfilter", "filter"),
++ },
++ videoConvert{
++ QGstElement::createFromFactory("videoconvert", "imageCaptureConvert"),
++ },
++ encoder{
++ QGstElement::createFromFactory("jpegenc", "jpegEncoder"),
++ },
++ muxer{
++ QGstElement::createFromFactory("jifmux", "jpegMuxer"),
++ },
++ sink{
++ QGstElement::createFromFactory("fakesink", "imageCaptureSink"),
++ }
++ {
++ // configures the queue to be fast, lightweight and non blocking
++ queue.set("leaky", 2 /*downstream*/);
++ queue.set("silent", true);
++ queue.set("max-size-buffers", int(1));
++ queue.set("max-size-bytes", int(0));
++ queue.set("max-size-time", uint64_t(0));
++
++ bin.add(queue, filter, videoConvert, encoder, muxer, sink);
++ qLinkGstElements(queue, filter, videoConvert, encoder, muxer, sink);
++ bin.addGhostPad(queue, "sink");
++
++ addProbeToPad(queue.staticPad("src").pad(), false);
++
++ sink.set("async", false);
++ }
++
++ QGstreamerImageCapture::~QGstreamerImageCapture()
++ {
++ bin.setStateSync(GST_STATE_NULL);
++
++ // wait for pending futures
++ auto pendingFutures = [&] {
++ QMutexLocker guard(&m_pendingFuturesMutex);
++ return std::move(m_pendingFutures);
++ }();
++
++ for (auto &element : pendingFutures)
++ element.second.waitForFinished();
++ }
++
++ bool QGstreamerImageCapture::isReadyForCapture() const
++ {
++ QMutexLocker guard(&m_mutex);
++ return m_session && !m_captureNextBuffer && cameraActive;
++ }
++
++ int QGstreamerImageCapture::capture(const QString &fileName)
++ {
++ using namespace Qt::Literals;
++ QString path = QMediaStorageLocation::generateFileName(
++ fileName, QStandardPaths::PicturesLocation, u"jpg"_s);
++ return doCapture(std::move(path));
++ }
++
++ int QGstreamerImageCapture::captureToBuffer()
++ {
++ return doCapture(QString());
++ }
++
++ int QGstreamerImageCapture::doCapture(QString fileName)
++ {
++ qCDebug(qLcImageCaptureGst) << "do capture";
++
++ {
++ QMutexLocker guard(&m_mutex);
++ if (!m_session) {
++ invokeDeferred([this] {
++ Q_EMIT error(-1, QImageCapture::ResourceError,
++ QPlatformImageCapture::msgImageCaptureNotSet());
++ });
++
++ qCDebug(qLcImageCaptureGst) << "error 1";
++ return -1;
++ }
++ if (!m_session->camera()) {
++ invokeDeferred([this] {
++ Q_EMIT error(-1, QImageCapture::ResourceError, tr("No camera available."));
++ });
++
++ qCDebug(qLcImageCaptureGst) << "error 2";
++ return -1;
++ }
++ if (m_captureNextBuffer) {
++ invokeDeferred([this] {
++ Q_EMIT error(-1, QImageCapture::NotReadyError,
++ QPlatformImageCapture::msgCameraNotReady());
++ });
++
++ qCDebug(qLcImageCaptureGst) << "error 3";
++ return -1;
++ }
++ m_lastId++;
++
++ pendingImages.enqueue({ m_lastId, std::move(fileName) });
++ // let one image pass the pipeline
++ m_captureNextBuffer = true;
++ }
++
++ Q_EMIT readyForCaptureChanged(false);
++ return m_lastId;
++ }
++
++ void QGstreamerImageCapture::saveBufferToFile(QGstBufferHandle buffer, QString filename, int taskId)
++ {
++ Q_ASSERT(!filename.isEmpty());
++
++ runInThreadPool(
++ [this, taskId, filename = std::move(filename), buffer = std::move(buffer)]() mutable {
++ QMutexLocker guard(&m_mutex);
++ qCDebug(qLcImageCaptureGst) << "saving image as" << filename;
++
++ QFile f(filename);
++ if (!f.open(QFile::WriteOnly)) {
++ qCDebug(qLcImageCaptureGst) << " could not open image file for writing";
++ return;
++ }
++
++ GstMapInfo info;
++ if (gst_buffer_map(buffer.get(), &info, GST_MAP_READ)) {
++ f.write(reinterpret_cast<const char *>(info.data), info.size);
++ gst_buffer_unmap(buffer.get(), &info);
++ }
++ f.close();
++
++ QMetaObject::invokeMethod(this, [this, taskId, filename = std::move(filename)]() mutable {
++ Q_EMIT imageSaved(taskId, filename);
++ });
++ });
++ }
++ void QGstreamerImageCapture::convertBufferToImage(const QMutexLocker<QRecursiveMutex> &locker,
++ QGstBufferHandle buffer, QGstCaps caps,
++ QMediaMetaData metadata, int taskId)
++ {
++ using namespace Qt::Literals;
++ Q_ASSERT(locker.mutex() == &m_mutex);
++ Q_ASSERT(locker.isLocked());
++
++ // QTBUG-131107: QVideoFrame::toImage() can only be called from the application thread
++ constexpr bool isOpenGLPlatform = QT_CONFIG(opengl);
++
++ // QTBUG-130970: QVideoFrame::toImage() on worker thread causes wayland to crash on the
++ // application thread
++ static const bool isWaylandQPA = QGuiApplication::platformName() == u"wayland"_s;
++
++ if (isOpenGLPlatform || isWaylandQPA) {
++ if (!m_session) {
++ qDebug() << "QGstreamerImageCapture::convertBufferToImage: no session";
++ return;
++ }
++
++ QGstVideoInfo previewInfo;
++ auto optionalVideoInfo = caps.videoInfo();
++ if (optionalVideoInfo)
++ previewInfo = *optionalVideoInfo;
++ QVideoFrame frame = qCreateFrameFromGstBuffer(buffer, previewInfo);
++
++ metadata.insert(QMediaMetaData::Resolution, frame.size());
++
++ invokeDeferred(
++ [this, frame = std::move(frame), taskId, metadata = std::move(metadata)]() mutable {
++ QImage img = frame.toImage();
++ if (img.isNull()) {
++ qDebug() << "received a null image";
++ return;
++ }
++
++ Q_EMIT imageExposed(taskId);
++ qCDebug(qLcImageCaptureGst) << "Image available!";
++ Q_EMIT imageAvailable(taskId, frame);
++ Q_EMIT imageCaptured(taskId, img);
++ Q_EMIT imageMetadataAvailable(taskId, metadata);
++ });
++ } else {
++ runInThreadPool([this, taskId, buffer = std::move(buffer), caps = std::move(caps),
++ metadata = std::move(metadata)]() mutable {
++ QMutexLocker guard(&m_mutex);
++ if (!m_session) {
++ qDebug() << "QGstreamerImageCapture::probeBuffer: no session";
++ return;
++ }
++
++ QGstVideoInfo previewInfo;
++ QVideoFrameFormat fmt;
++ auto optionalVideoInfo = caps.videoInfo();
++ if (optionalVideoInfo) {
++ previewInfo = *optionalVideoInfo;
++ fmt = qVideoFrameFormatFromGstVideoInfo(previewInfo);
++ }
++
++ QVideoFrame frame = qCreateFrameFromGstBuffer(std::move(buffer), previewInfo, fmt);
++ QImage img = frame.toImage();
++ if (img.isNull()) {
++ qDebug() << "received a null image";
++ return;
++ }
++
++ QMediaMetaData imageMetaData = metaData();
++ imageMetaData.insert(QMediaMetaData::Resolution, frame.size());
++
++ invokeDeferred([this, taskId, metadata = std::move(metadata), frame = std::move(frame),
++ img = std::move(img)]() mutable {
++ Q_EMIT imageExposed(taskId);
++ qCDebug(qLcImageCaptureGst) << "Image available!";
++ Q_EMIT imageAvailable(taskId, frame);
++ Q_EMIT imageCaptured(taskId, img);
++ Q_EMIT imageMetadataAvailable(taskId, metadata);
++ });
++ });
++ }
++ }
++
++ void QGstreamerImageCapture::setResolution(const QSize &resolution)
++ {
++ QGstCaps padCaps = bin.staticPad("sink").currentCaps();
++ if (!padCaps) {
++ qDebug() << "Camera not ready";
++ return;
++ }
++ QGstCaps caps = padCaps.copy();
++ if (!caps)
++ return;
++
++ gst_caps_set_simple(caps.caps(), "width", G_TYPE_INT, resolution.width(), "height", G_TYPE_INT,
++ resolution.height(), nullptr);
++ filter.set("caps", caps);
++ }
++
++ bool QGstreamerImageCapture::probeBuffer(GstBuffer *buffer)
++ {
++ if (!m_captureNextBuffer.load())
++ return false;
++
++ QMutexLocker guard(&m_mutex);
++ qCDebug(qLcImageCaptureGst) << "probe buffer";
++
++ QGstBufferHandle bufferHandle{
++ buffer,
++ QGstBufferHandle::NeedsRef,
++ };
++
++ m_captureNextBuffer = false;
++
++ bool ready = isReadyForCapture();
++ invokeDeferred([this, ready] {
++ Q_EMIT readyForCaptureChanged(ready);
++ });
++
++ // save file
++ PendingImage imageData = pendingImages.dequeue();
++ QString saveFileName = imageData.filename;
++ if (!saveFileName.isEmpty())
++ saveBufferToFile(bufferHandle, std::move(saveFileName), imageData.id);
++
++ // convert to image and emit
++ QGstCaps caps = bin.staticPad("sink").currentCaps();
++ QMediaMetaData imageMetaData = metaData();
++ convertBufferToImage(guard, bufferHandle, std::move(caps), std::move(imageMetaData),
++ imageData.id);
++
++ return true;
++ }
++
++ void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session)
++ {
++ QMutexLocker guard(&m_mutex);
++ QGstreamerMediaCaptureSession *captureSession = static_cast<QGstreamerMediaCaptureSession *>(session);
++ if (m_session == captureSession)
++ return;
++
++ bool readyForCapture = isReadyForCapture();
++ if (m_session) {
++ disconnect(m_session, nullptr, this, nullptr);
++ m_lastId = 0;
++ pendingImages.clear();
++ m_captureNextBuffer = false;
++ cameraActive = false;
++ }
++
++ m_session = captureSession;
++ if (!m_session) {
++ if (readyForCapture)
++ Q_EMIT readyForCaptureChanged(false);
++ return;
++ }
++
++ connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this,
++ &QGstreamerImageCapture::onCameraChanged);
++ onCameraChanged();
++ }
++
++ void QGstreamerImageCapture::setMetaData(const QMediaMetaData &m)
++ {
++ {
++ QMutexLocker guard(&m_mutex);
++ QPlatformImageCapture::setMetaData(m);
++ }
++
++ // ensure taginject injects this metaData
++ applyMetaDataToTagSetter(m, muxer);
++ }
++
++ void QGstreamerImageCapture::cameraActiveChanged(bool active)
++ {
++ qCDebug(qLcImageCaptureGst) << "cameraActiveChanged" << cameraActive << active;
++ if (cameraActive == active)
++ return;
++ cameraActive = active;
++ qCDebug(qLcImageCaptureGst) << "isReady" << isReadyForCapture();
++ Q_EMIT readyForCaptureChanged(isReadyForCapture());
++ }
++
++ void QGstreamerImageCapture::onCameraChanged()
++ {
++ QMutexLocker guard(&m_mutex);
++ if (m_session->camera()) {
++ cameraActiveChanged(m_session->camera()->isActive());
++ connect(m_session->camera(), &QPlatformCamera::activeChanged, this,
++ &QGstreamerImageCapture::cameraActiveChanged);
++ } else {
++ cameraActiveChanged(false);
++ }
++ }
++
++ QImageEncoderSettings QGstreamerImageCapture::imageSettings() const
++ {
++ return m_settings;
++ }
++
++ void QGstreamerImageCapture::setImageSettings(const QImageEncoderSettings &settings)
++ {
++ if (m_settings != settings) {
++ QSize resolution = settings.resolution();
++ if (m_settings.resolution() != resolution && !resolution.isEmpty())
++ setResolution(resolution);
++
++ m_settings = settings;
++ }
++ }
++
++ QT_END_NAMESPACE
++
++ #include "moc_qgstreamerimagecapture_p.cpp"