Revision control
+ // Copyright (C) 2026 The Qt Company Ltd.
+ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
+
+ #include <QtMultimediaTestLib/private/qfileutil_p.h>
+ #include <QtMultimediaTestLib/private/rhi_support_p.h>
+ #include <QtTest/qtest.h>
+
+ #include <QtFFmpegMediaPluginImpl/private/qffmpeg_p.h>
+ #include <QtFFmpegMediaPluginImpl/private/qffmpeghwaccel_mediacodec_p.h>
+ #include <QtFFmpegMediaPluginImpl/private/qffmpeghwaccel_p.h>
+ #include <QtFFmpegMediaPluginImpl/private/qffmpegvideobuffer_p.h>
+
+ #include <QtMultimedia/qvideoframe.h>
+ #include <QtMultimedia/private/qvideoframe_p.h>
+
+ #include <QtGui/qguiapplication.h>
+ #include <QtGui/qoffscreensurface.h>
+ #include <QtGui/rhi/qrhi.h>
+
+ #include <QtCore/qfileinfo.h>
+ #include <QtCore/qregularexpression.h>
+ #include <QtCore/qtemporaryfile.h>
+ #include <QtCore/qurl.h>
+
+ extern "C" {
+ #include <libavcodec/avcodec.h>
+ #include <libavformat/avformat.h>
+ }
+
+ #include <chrono>
+ #include <memory>
+ #include <optional>
+
+ using namespace QFFmpeg;
+ using namespace QtMultimediaTest;
+ using namespace Qt::Literals;
+ using namespace std::chrono_literals;
+
+ namespace {
+
+ // MediaCodec's AVMediaCodecBuffer-backed AVFrame can only come from a real, running AMediaCodec decode.
+ class MediaCodecDeviceContext
+ {
+ public:
+ static std::optional<MediaCodecDeviceContext> create(const QUrl &url)
+ {
+ MediaCodecDeviceContext context;
+
+ QString realFilePath;
+ if (url.scheme() == "qrc"_L1) {
+ const QString suffix = QFileInfo(url.path()).suffix();
+ auto temporaryFile = copyResourceToTemporaryFile(url, u"XXXXXX."_s + suffix);
+ if (!temporaryFile)
+ return std::nullopt;
+
+ realFilePath = temporaryFile->fileName();
+ context.m_extractedResourceFile = std::move(temporaryFile);
+ } else if (url.isLocalFile()) {
+ realFilePath = url.toLocalFile();
+ } else {
+ return std::nullopt;
+ }
+
+ AVFormatContext *rawFormatContext = nullptr;
+ const QByteArray path = realFilePath.toUtf8();
+ if (avformat_open_input(&rawFormatContext, path.constData(), nullptr, nullptr) < 0)
+ return std::nullopt;
+ context.m_formatContext.reset(rawFormatContext);
+
+ if (avformat_find_stream_info(context.m_formatContext.get(), nullptr) < 0)
+ return std::nullopt;
+
+ const AVCodec *unusedCodec = nullptr;
+ const int streamIndex = av_find_best_stream(context.m_formatContext.get(),
+ AVMEDIA_TYPE_VIDEO, -1, -1, &unusedCodec, 0);
+ if (streamIndex < 0)
+ return std::nullopt;
+ context.m_videoStreamIndex = streamIndex;
+
+ const AVCodecParameters *codecParams =
+ context.m_formatContext->streams[*context.m_videoStreamIndex]->codecpar;
+
+ auto [codec, hwAccel] = HWAccel::findDecoderWithHwAccel(codecParams->codec_id);
+ if (!codec || !hwAccel || hwAccel->deviceType() != AV_HWDEVICE_TYPE_MEDIACODEC)
+ return std::nullopt;
+ context.m_hwAccel = std::move(hwAccel);
+
+ context.m_codecContext.reset(avcodec_alloc_context3(codec->get()));
+ if (!context.m_codecContext)
+ return std::nullopt;
+
+ if (avcodec_parameters_to_context(context.m_codecContext.get(), codecParams) < 0)
+ return std::nullopt;
+
+ context.m_codecContext->hw_device_ctx =
+ av_buffer_ref(context.m_hwAccel->hwDeviceContextAsBuffer());
+ context.m_codecContext->get_format = &getFormat;
+
+ if (avcodec_open2(context.m_codecContext.get(), codec->get(), nullptr) < 0)
+ return std::nullopt;
+
+ return context;
+ }
+
+ MediaCodecDeviceContext(MediaCodecDeviceContext &&) noexcept = default;
+ MediaCodecDeviceContext &operator=(MediaCodecDeviceContext &&) noexcept = default;
+
+ AVFrameUPtr decodeNextFrame()
+ {
+ AVFrameUPtr frame = makeAVFrame();
+ AVPacketUPtr packet(av_packet_alloc());
+
+ while (true) {
+ const int receiveResult = avcodec_receive_frame(m_codecContext.get(), frame.get());
+ if (receiveResult == 0) {
+ // QTBUG-108446: the MediaCodec decoder never attaches an AVHWFramesContext to its
+ // output frames (compare qffmpegvideorenderer.cpp)
+ if (!frame->hw_frames_ctx) {
+ if (!m_hwAccel->hwFramesContext())
+ m_hwAccel->createFramesContext(AVPixelFormat(frame->format),
+ { frame->width, frame->height });
+ if (m_hwAccel->hwFramesContext())
+ frame->hw_frames_ctx = av_buffer_ref(m_hwAccel->hwFramesContextAsBuffer());
+ }
+ Q_ASSERT(frame->format == AV_PIX_FMT_MEDIACODEC);
+ return frame;
+ }
+ if (receiveResult != AVERROR(EAGAIN))
+ return nullptr; // EOF or a real decode error
+
+ int readResult = 0;
+ do {
+ av_packet_unref(packet.get());
+ readResult = av_read_frame(m_formatContext.get(), packet.get());
+ } while (readResult == 0 && packet->stream_index != *m_videoStreamIndex);
+
+ if (readResult < 0) {
+ // Flush: signal EOF to the decoder and let the next receive_frame() drain it.
+ avcodec_send_packet(m_codecContext.get(), nullptr);
+ continue;
+ }
+
+ if (avcodec_send_packet(m_codecContext.get(), packet.get()) < 0)
+ return nullptr;
+ }
+ }
+
+ private:
+ MediaCodecDeviceContext() = default;
+
+ AVDemuxerContextUPtr m_formatContext;
+ AVCodecContextUPtr m_codecContext;
+ HWAccelUPtr m_hwAccel;
+ std::optional<int> m_videoStreamIndex;
+ // Kept alive for the object's lifetime when filePath was a resource path; the demuxer reads
+ // from this file lazily throughout decodeNextFrame(), not just at open time.
+ std::unique_ptr<QTemporaryFile> m_extractedResourceFile;
+ };
+
+ } // namespace
+
+ class tst_QMediaCodecTextureConverter : public QObject
+ {
+ Q_OBJECT
+
+ private slots:
+ void create_succeedsAndKeepsRhi_whenGlIsAvailable();
+ void createTextureHandles_returnsRedTexture_forRealDecodedFrame_data();
+ void createTextureHandles_returnsRedTexture_forRealDecodedFrame();
+ void createTextureHandles_reusesTexture_onSecondRealFrame();
+ void createTextureHandles_returnsNull_forFrameWithoutHwFramesCtx();
+ void map_returnsFalse_forRealDecodedFrame();
+
+ private:
+ // The MediaCodec decode is asynchronous: av_mediacodec_release_buffer() only requests that
+ // the buffer be rendered to the Surface, and AndroidSurfaceTexture::updateTexImage() can
+ // legitimately be a no-op if that render hasn't landed yet. This is an inherent property of
+ // the SurfaceTexture API, not test flakiness -- poll with a bounded retry instead of asserting
+ // on the first attempt.
+ std::optional<QByteArray> pollReadback(QRhi &rhi, quint64 textureHandle, QSize size);
+ };
+
+ std::optional<QByteArray>
+ tst_QMediaCodecTextureConverter::pollReadback(QRhi &rhi, quint64 textureHandle, QSize size)
+ {
+ constexpr int maxAttempts = 50;
+ constexpr auto retryDelay = 20ms;
+ for (int attempt = 0; attempt < maxAttempts; ++attempt) {
+ auto result = readBackExternalOesTexture(rhi, textureHandle, size);
+ if (result)
+ return *result;
+ QTest::qWait(retryDelay);
+ }
+ return std::nullopt;
+ }
+
+ void tst_QMediaCodecTextureConverter::create_succeedsAndKeepsRhi_whenGlIsAvailable()
+ {
+ q23::expected<OffscreenGlRhi, QString> offscreenRhi = createOffscreenGlRhi();
+ if (!offscreenRhi)
+ QSKIP(qPrintable(u"Could not create an OpenGL ES2 QRhi backend on this system: "_s
+ + offscreenRhi.error()));
+ auto &[fallbackSurface, rhi] = *offscreenRhi;
+
+ std::shared_ptr converter = MediaCodecTextureConverter::create(rhi.get());
+ QVERIFY(converter);
+ QCOMPARE(converter->rhi, rhi.get());
+ }
+
+ void tst_QMediaCodecTextureConverter::
+ createTextureHandles_returnsRedTexture_forRealDecodedFrame_data()
+ {
+ QTest::addColumn<QUrl>("sourceUrl");
+
+ QTest::newRow("NV12") << QUrl(u"qrc:/testdata/one_red_frame.mp4"_s);
+ QTest::newRow("P010") << QUrl(u"qrc:/testdata/one_red_frame_p010.mp4"_s);
+ }
+
+ void tst_QMediaCodecTextureConverter::createTextureHandles_returnsRedTexture_forRealDecodedFrame()
+ {
+ QFETCH(QUrl, sourceUrl);
+
+ q23::expected<OffscreenGlRhi, QString> offscreenRhi = createOffscreenGlRhi();
+ if (!offscreenRhi)
+ QSKIP(qPrintable(u"Could not create an OpenGL ES2 QRhi backend on this system: "_s
+ + offscreenRhi.error()));
+ auto &[fallbackSurface, rhi] = *offscreenRhi;
+
+ // AndroidSurfaceTexture's constructor calls the deprecated android.graphics.SurfaceTexture(int
+ // texName) constructor, which requires a GL context to be current on the calling thread; this
+ // must be current before MediaCodecTextureConverter::setupDecoderSurface() runs, which happens
+ // synchronously inside avcodec_open2()/decodeNextFrame() via the get_format callback -- not
+ // when createTextureHandles() is called afterwards.
+ rhi->makeThreadLocalNativeContextCurrent();
+
+ std::optional<MediaCodecDeviceContext> deviceContext =
+ MediaCodecDeviceContext::create(sourceUrl);
+ if (!deviceContext)
+ QSKIP("Could not open a MediaCodec hw decode session on this device");
+
+ AVFrameUPtr frame = deviceContext->decodeNextFrame();
+ QVERIFY(frame);
+ QCOMPARE(frame->format, AV_PIX_FMT_MEDIACODEC);
+
+ std::shared_ptr converter = MediaCodecTextureConverter::create(rhi.get());
+ QVideoFrameTexturesHandlesUPtr handles = converter->createTextureHandles(frame.get(), nullptr);
+ QVERIFY(handles);
+
+ const quint64 textureHandle = handles->textureHandle(*rhi, 0);
+ QVERIFY(textureHandle != 0);
+
+ const QSize frameSize(frame->width, frame->height);
+ std::optional<QByteArray> pixels = pollReadback(*rhi, textureHandle, frameSize);
+ if (!pixels)
+ QFAIL("Timed out waiting for the decoded buffer to be rendered to the surface");
+
+ // Sanity-check a handful of pixels rather than the whole frame: exact color reproduction
+ // through a real hw video decoder is not bit-exact (chroma subsampling/color conversion), so
+ // only assert "clearly red", not an exact RGBA match.
+ const int stride = frameSize.width() * 4;
+ for (int y : { 0, frameSize.height() / 2, frameSize.height() - 1 }) {
+ const uchar *pixel = reinterpret_cast<const uchar *>(pixels->constData() + y * stride);
+ QVERIFY2(pixel[0] > 150 && pixel[1] < 100 && pixel[2] < 100,
+ qPrintable(u"Pixel at row %1 was not clearly red: (%2, %3, %4)"_s.arg(y)
+ .arg(pixel[0])
+ .arg(pixel[1])
+ .arg(pixel[2])));
+ }
+ }
+
+ void tst_QMediaCodecTextureConverter::createTextureHandles_reusesTexture_onSecondRealFrame()
+ {
+ q23::expected<OffscreenGlRhi, QString> offscreenRhi = createOffscreenGlRhi();
+ if (!offscreenRhi)
+ QSKIP(qPrintable(u"Could not create an OpenGL ES2 QRhi backend on this system: "_s
+ + offscreenRhi.error()));
+ auto &[fallbackSurface, rhi] = *offscreenRhi;
+
+ rhi->makeThreadLocalNativeContextCurrent();
+
+ std::optional<MediaCodecDeviceContext> deviceContext =
+ MediaCodecDeviceContext::create(QUrl(u"qrc:/testdata/colors.mp4"_s));
+ if (!deviceContext)
+ QSKIP("Could not open a MediaCodec hw decode session on this device");
+
+ std::shared_ptr converter = MediaCodecTextureConverter::create(rhi.get());
+
+ AVFrameUPtr firstFrame = deviceContext->decodeNextFrame();
+ QVERIFY(firstFrame);
+ QVideoFrameTexturesHandlesUPtr firstHandles =
+ converter->createTextureHandles(firstFrame.get(), nullptr);
+ QVERIFY(firstHandles);
+ const quint64 firstHandle = firstHandles->textureHandle(*rhi, 0);
+ QVERIFY(firstHandle != 0);
+
+ AVFrameUPtr secondFrame = deviceContext->decodeNextFrame();
+ QVERIFY(secondFrame);
+ QVideoFrameTexturesHandlesUPtr secondHandles =
+ converter->createTextureHandles(secondFrame.get(), std::move(firstHandles));
+ QVERIFY(secondHandles);
+ const quint64 secondHandle = secondHandles->textureHandle(*rhi, 0);
+ QVERIFY(secondHandle != 0);
+
+ // Same decoder Surface/AndroidSurfaceTexture across both calls, so the underlying QRhiTexture
+ // is expected to be reused rather than recreated (only invalidated when
+ // AndroidSurfaceTexture::index() changes, which doesn't happen here).
+ QCOMPARE(secondHandle, firstHandle);
+ }
+
+ void tst_QMediaCodecTextureConverter::map_returnsFalse_forRealDecodedFrame()
+ {
+ std::optional<MediaCodecDeviceContext> deviceContext =
+ MediaCodecDeviceContext::create(QUrl(u"qrc:/testdata/one_red_frame.mp4"_s));
+ if (!deviceContext)
+ QSKIP("Could not open a MediaCodec hw decode session on this device");
+
+ AVFrameUPtr frame = deviceContext->decodeNextFrame();
+ QVERIFY(frame);
+ QCOMPARE(frame->format, AV_PIX_FMT_MEDIACODEC);
+
+ auto buffer = std::make_unique<QFFmpegVideoBuffer>(std::move(frame));
+ QVideoFrameFormat format(buffer->size(), buffer->pixelFormat());
+ QVideoFrame videoFrame = QVideoFramePrivate::createFrame(std::move(buffer), format);
+
+ QTest::ignoreMessage(QtWarningMsg,
+ QRegularExpression(u"Error transferring the data to system memory.*"_s));
+ QVERIFY(!videoFrame.map(QVideoFrame::ReadOnly));
+ }
+
+ void tst_QMediaCodecTextureConverter::createTextureHandles_returnsNull_forFrameWithoutHwFramesCtx()
+ {
+ q23::expected<OffscreenGlRhi, QString> offscreenRhi = createOffscreenGlRhi();
+ if (!offscreenRhi)
+ QSKIP(qPrintable(u"Could not create an OpenGL ES2 QRhi backend on this system: "_s
+ + offscreenRhi.error()));
+ auto &[fallbackSurface, rhi] = *offscreenRhi;
+
+ std::shared_ptr converter = MediaCodecTextureConverter::create(rhi.get());
+
+ AVFrameUPtr frame = makeAVFrame();
+ frame->width = 64;
+ frame->height = 64;
+ frame->format = AV_PIX_FMT_YUV420P;
+ // Deliberately no hw_frames_ctx: getTextureSurface() bails out on that check alone, so this
+ // needs no real decode.
+
+ QVideoFrameTexturesHandlesUPtr handles = converter->createTextureHandles(frame.get(), nullptr);
+ QVERIFY(!handles);
+ }
+
+ QTEST_MAIN(tst_QMediaCodecTextureConverter)
+
+ #include "tst_qmediacodectextureconverter.moc"