Revision control

++ // Copyright (C) 2024 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
++
++ #include <QtMultimediaTestLib/private/mediabackendutils_p.h>
++ #include <QtMultimediaTestLib/private/mediafileselector_p.h>
++ #include <QtMultimediaTestLib/private/osdetection_p.h>
++ #include <QtMultimediaTestLib/private/qintegrationtestbase_p.h>
++ #include <QtMultimediaTestLib/private/testvideosink_p.h>
++ #include <QtTest/qtest.h>
++ #include <QtMultimedia/qmediaplayer.h>
++ #include <QtMultimedia/qvideoframe.h>
++ #include <QtMultimedia/private/qthreadlocalrhi_p.h>
++ #include <QtMultimedia/private/qvideotexturehelper_p.h>
++ #include <QtMultimedia/private/qvideowindow_p.h>
++ #include <QtGui/rhi/qrhi.h>
++ #include <QtCore/qdebug.h>
++ #include <QtCore/qthreadpool.h>
++
++ #include <list>
++
++ using namespace QtMultimediaTest;
++
++ class tst_QVideoFrameBackend : public QIntegrationTestBase
++ {
++ Q_OBJECT
++
++ public slots:
++ void initTestCase();
++
++ private slots:
++ void testMediaFilesAreSupported();
++
++ void maxLuminance_isTheMasteringDisplaysPeakInNits_data();
++ void maxLuminance_isTheMasteringDisplaysPeakInNits();
++
++ void toImage_retainsThePreviousMappedState_data();
++ void toImage_retainsThePreviousMappedState();
++
++ void toImage_rendersUpdatedFrame_afterMappingInWriteModeAndModifying_data();
++ void toImage_rendersUpdatedFrame_afterMappingInWriteModeAndModifying();
++
++ void toImage_returnsImage_whenCalledFromSeparateThreadAndWhileRenderingToWindow_data();
++ void toImage_returnsImage_whenCalledFromSeparateThreadAndWhileRenderingToWindow();
++
++ private:
++ QVideoFrame createDefaultFrame() const;
++
++ QVideoFrame createMediaPlayerFrame() const;
++
++ using FrameCreator = decltype(&tst_QVideoFrameBackend::createDefaultFrame);
++
++ template <typename F>
++ void addMediaPlayerFrameTestData(F &&f);
++
++ private:
++ MaybeUrl m_oneRedFrameVideo{ q23::unexpect };
++ MaybeUrl m_colorsVideo{ q23::unexpect };
++ MaybeUrl m_masteringVideo{ q23::unexpect };
++ MediaFileSelector m_mediaSelector;
++ };
++
++ QVideoFrame tst_QVideoFrameBackend::createDefaultFrame() const
++ {
++ return QVideoFrame(QVideoFrameFormat(QSize(10, 20), QVideoFrameFormat::Format_ARGB8888));
++ }
++
++ QVideoFrame tst_QVideoFrameBackend::createMediaPlayerFrame() const
++ {
++ if (!m_oneRedFrameVideo)
++ return {};
++
++ TestVideoSink sink;
++ QMediaPlayer player;
++
++ player.setVideoOutput(&sink);
++ player.setSource(*m_oneRedFrameVideo);
++
++ player.play();
++
++ return sink.waitForFrame();
++ }
++
++ template <typename F>
++ void tst_QVideoFrameBackend::addMediaPlayerFrameTestData(F &&f)
++ {
++ if (!m_oneRedFrameVideo) {
++ qWarning() << "Skipping test data with mediaplayer as the source cannot be open."
++ "\nSee the test case 'testMediaFilesAreSupported' for details";
++ return;
++ }
++
++ if (isGStreamerPlatform()) {
++ qWarning() << "createMediaPlayerFrame spuriously fails with gstreamer";
++ return;
++ }
++
++ if (isOhosPlatform()) {
++ // createMediaPlayerFrame returns an invalid frame on OHOS because the
++ // OHOS video output cannot deliver textures into a bare QVideoSink
++ // that has no live RHI.
++ qWarning() << "createMediaPlayerFrame returns no frame on OHOS";
++ return;
++ }
++
++ f();
++ }
++
++ void tst_QVideoFrameBackend::initTestCase()
++ {
++ if (!initIntegrationTestCase())
++ return;
++
++ #ifdef Q_OS_ANDROID
++ qWarning() << "Skip media selection, QTBUG-118571";
++ return;
++ #endif
++
++ m_oneRedFrameVideo = m_mediaSelector.select("qrc:/testdata/one_red_frame.mp4");
++ m_colorsVideo = m_mediaSelector.select("qrc:/testdata/colors.mp4");
++ m_masteringVideo = m_mediaSelector.select("qrc:/testdata/mastering_1000nits.mp4");
++ }
++
++ void tst_QVideoFrameBackend::testMediaFilesAreSupported()
++ {
++ #ifdef Q_OS_ANDROID
++ QSKIP("Skip test cases with mediaPlayerFrame on Android CI, because of QTBUG-118571");
++ #endif
++ #ifdef Q_OS_HARMONY
++ QSKIP("OHOS demuxer rejects some H.264/HEVC/AV1 profiles used by the test media");
++ #endif
++ if (isWMFPlatform())
++ QSKIP("WMF backend cannot mastering_1000nits.mp4");
++
++ QCOMPARE(m_mediaSelector.dumpErrors(), "");
++ }
++
++ void tst_QVideoFrameBackend::maxLuminance_isTheMasteringDisplaysPeakInNits_data()
++ {
++ #ifdef Q_OS_ANDROID
++ QSKIP("Skip test cases with mediaPlayerFrame on Android CI, because of QTBUG-118571");
++ #endif
++ QTest::addColumn<QUrl>("source");
++ QTest::addColumn<float>("expectedMaxLuminance");
++
++ // Files without mastering display metadata fall back to the SDR peak
++ if (m_oneRedFrameVideo)
++ QTest::newRow("one_red_frame") << *m_oneRedFrameVideo << 100.f;
++ if (m_colorsVideo)
++ QTest::newRow("colors") << *m_colorsVideo << 100.f;
++
++ // HEVC with the SMPTE ST 2086 mastering display metadata of x265's
++ // --master-display "...L(10000000,1)": a peak of 1000 cd/m2 (the unit of the frame's
++ // AVMasteringDisplayMetadata, a rational that is 10000000/10000 here)
++ if (m_masteringVideo)
++ QTest::newRow("mastering_1000nits") << *m_masteringVideo << 1000.f;
++ }
++
++ void tst_QVideoFrameBackend::maxLuminance_isTheMasteringDisplaysPeakInNits()
++ {
++ QFETCH(const QUrl, source);
++ QFETCH(const float, expectedMaxLuminance);
++
++ if (!isFFMPEGPlatform())
++ QSKIP("Only the FFmpeg backend reads the mastering display metadata");
++
++ TestVideoSink sink;
++ QMediaPlayer player;
++ player.setVideoOutput(&sink);
++ player.setSource(source);
++ player.play();
++
++ const QVideoFrame frame = sink.waitForFrame();
++ QVERIFY(frame.isValid());
++ QCOMPARE(frame.surfaceFormat().maxLuminance(), expectedMaxLuminance);
++ }
++
++ void tst_QVideoFrameBackend::toImage_retainsThePreviousMappedState_data()
++ {
++ QTest::addColumn<FrameCreator>("frameCreator");
++ QTest::addColumn<QVideoFrame::MapMode>("initialMapMode");
++
++ // clang-format off
++ QTest::addRow("defaulFrame.notMapped") << &tst_QVideoFrameBackend::createDefaultFrame
++ << QVideoFrame::NotMapped;
++ QTest::addRow("defaulFrame.readOnly") << &tst_QVideoFrameBackend::createDefaultFrame
++ << QVideoFrame::ReadOnly;
++
++ addMediaPlayerFrameTestData([]()
++ {
++ QTest::addRow("mediaPlayerFrame.notMapped")
++ << &tst_QVideoFrameBackend::createMediaPlayerFrame
++ << QVideoFrame::NotMapped;
++ QTest::addRow("mediaPlayerFrame.readOnly")
++ << &tst_QVideoFrameBackend::createMediaPlayerFrame
++ << QVideoFrame::ReadOnly;
++ });
++
++ // clang-format on
++ }
++
++ void tst_QVideoFrameBackend::toImage_retainsThePreviousMappedState()
++ {
++ QFETCH(const FrameCreator, frameCreator);
++ QFETCH(const QVideoFrame::MapMode, initialMapMode);
++ const bool initiallyMapped = initialMapMode != QVideoFrame::NotMapped;
++
++ QVideoFrame frame = std::invoke(frameCreator, this);
++ QVERIFY(frame.isValid());
++
++ frame.map(initialMapMode);
++ QCOMPARE(frame.mapMode(), initialMapMode);
++
++ QImage image = frame.toImage();
++ QVERIFY(!image.isNull());
++
++ QCOMPARE(frame.mapMode(), initialMapMode);
++ QCOMPARE(frame.isMapped(), initiallyMapped);
++ }
++
++ void tst_QVideoFrameBackend::toImage_rendersUpdatedFrame_afterMappingInWriteModeAndModifying_data()
++ {
++ QTest::addColumn<FrameCreator>("frameCreator");
++ QTest::addColumn<QVideoFrame::MapMode>("mapMode");
++
++ // clang-format off
++ QTest::addRow("defaulFrame.writeOnly") << &tst_QVideoFrameBackend::createDefaultFrame
++ << QVideoFrame::WriteOnly;
++ QTest::addRow("defaulFrame.readWrite") << &tst_QVideoFrameBackend::createDefaultFrame
++ << QVideoFrame::ReadWrite;
++
++ addMediaPlayerFrameTestData([]()
++ {
++ QTest::addRow("mediaPlayerFrame.writeOnly")
++ << &tst_QVideoFrameBackend::createMediaPlayerFrame
++ << QVideoFrame::WriteOnly;
++ QTest::addRow("mediaPlayerFrame.readWrite")
++ << &tst_QVideoFrameBackend::createMediaPlayerFrame
++ << QVideoFrame::ReadWrite;
++ });
++ // clang-format on
++ }
++
++ void tst_QVideoFrameBackend::toImage_rendersUpdatedFrame_afterMappingInWriteModeAndModifying()
++ {
++ QFETCH(const FrameCreator, frameCreator);
++ QFETCH(const QVideoFrame::MapMode, mapMode);
++
++ if (isWMFPlatform())
++ // TODO: WMF backend only supports WriteOnly mapping
++ QSKIP("WMF backend does not support WriteOnly mapping");
++
++ // Arrange
++
++ QVideoFrame frame = std::invoke(frameCreator, this);
++ QVERIFY(frame.isValid());
++
++ QImage originalImage = frame.toImage();
++ QVERIFY(!originalImage.isNull());
++
++ // Act: map the frame in write mode and change the top level pixel
++ frame.map(mapMode);
++ QVERIFY(frame.isWritable());
++
++ QCOMPARE_NE(frame.pixelFormat(), QVideoFrameFormat::Format_Invalid);
++
++ const QVideoTextureHelper::TextureDescription *textureDescription =
++ QVideoTextureHelper::textureDescription(frame.pixelFormat());
++ QVERIFY(textureDescription);
++
++ uchar *firstPlaneBits = frame.bits(0);
++ QVERIFY(firstPlaneBits);
++
++ for (int i = 0; i < textureDescription->strideFactor; ++i)
++ firstPlaneBits[i] = ~firstPlaneBits[i];
++
++ frame.unmap();
++
++ // get an image from modified frame
++ QImage modifiedImage = frame.toImage();
++
++ // Assert
++
++ QVERIFY(!frame.isMapped());
++ QCOMPARE_NE(originalImage.pixel(0, 0), modifiedImage.pixel(0, 0));
++ QCOMPARE(originalImage.pixel(1, 0), modifiedImage.pixel(1, 0));
++ QCOMPARE(originalImage.pixel(1, 1), modifiedImage.pixel(1, 1));
++ }
++
++ void tst_QVideoFrameBackend::toImage_returnsImage_whenCalledFromSeparateThreadAndWhileRenderingToWindow_data()
++ {
++ QTest::addColumn<QRhi::Implementation>("backend");
++
++ #if QT_CONFIG(opengl)
++ QTest::newRow("OpenGL") << QRhi::OpenGLES2;
++ #endif
++ #if QT_CONFIG(metal)
++ QTest::newRow("Metal") << QRhi::Metal;
++ #endif
++ #if defined(Q_OS_WIN)
++ QTest::newRow("D3D11") << QRhi::D3D11;
++ #endif
++ }
++
++ void tst_QVideoFrameBackend::toImage_returnsImage_whenCalledFromSeparateThreadAndWhileRenderingToWindow()
++ {
++ QFETCH(QRhi::Implementation, backend);
++
++ #ifdef Q_OS_HARMONY
++ QSKIP("OHOS demuxer rejects the H.264 profile used by colors.mp4");
++ #endif
++
++ if (isCI()) {
++ #ifdef Q_OS_MACOS
++ if (backend == QRhi::Metal)
++ QSKIP("SKIP on macOS because of crash and error \"Failed to create QWindow::MetalSurface. Metal is not supported by any of the GPUs in this system.\"");
++ #elif defined(Q_OS_ANDROID)
++ QSKIP("SKIP initTestCase on CI, because of QTBUG-118571");
++ #endif
++ }
++ // Arrange
++ QVideoWindow window;
++ window.show();
++
++ QVERIFY(QTest::qWaitForWindowExposed(&window));
++
++ QMediaPlayer player;
++ player.setVideoOutput(&window);
++
++ const QVideoSink *sink = window.videoSink();
++ std::list<QImage> images;
++
++ QThreadPool threadPool;
++ threadPool.setMaxThreadCount(4);
++
++ // act
++ connect(sink, &QVideoSink::videoFrameChanged, sink, [&](const QVideoFrame &frame) {
++ if (!frame.isValid())
++ return; // ignore last frame
++
++ images.push_back({});
++
++ threadPool.start([it = std::prev(images.end()), frame, backend]() {
++ qSetPreferredThreadLocalRhiBackend(backend);
++ *it = frame.toImage();
++ });
++ });
++
++ // Arrange some more
++ player.setSource(*m_colorsVideo);
++ player.setLoops(5);
++ player.setPlaybackRate(5.f);
++ player.play();
++
++ QTRY_COMPARE_GE_WITH_TIMEOUT(images.size(), 25u, std::chrono::seconds(60));
++ player.stop();
++
++ QVERIFY(threadPool.waitForDone(std::chrono::seconds(60)));
++
++ const size_t validImagesCount = std::count_if(
++ images.begin(), images.end(), [](const QImage &image) { return !image.isNull(); });
++
++ QCOMPARE(validImagesCount, images.size());
++ }
++
++ QTEST_MAIN(tst_QVideoFrameBackend)
++ #include "tst_qvideoframebackend.moc"