qffmpegvideobuffer.cpp

++ // Copyright (C) 2021 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++

++ #include "qffmpegvideobuffer_p.h"

++ #include "private/qvideotexturehelper_p.h"

++ #include "private/qmultimediautils_p.h"

++ #include "qffmpeghwaccel_p.h"

++ #include "qloggingcategory.h"

++ #include <QtCore/qthread.h>

++

++ extern "C" {

++ #include <libavutil/pixdesc.h>

++ #include <libavutil/hdr_dynamic_metadata.h>

++ #include <libavutil/mastering_display_metadata.h>

++ }

++

++ QT_BEGIN_NAMESPACE

++

++ using namespace QFFmpeg;

++

++ Q_STATIC_LOGGING_CATEGORY(qLcFFmpegVideoBuffer, "qt.multimedia.ffmpeg.videobuffer");

++

++ static bool isFrameFlipped(const AVFrame& frame) {

++     for (int i = 0; i < AV_NUM_DATA_POINTERS && frame.data[i]; ++i) {

++         if (frame.linesize[i] < 0)

++             return true;

++     }

++

++     return false;

++ }

++

++ QFFmpegVideoBuffer::QFFmpegVideoBuffer(AVFrameUPtr frame, AVRational pixelAspectRatio)

++     : m_frame(frame.get()),

++       m_size(qCalculateFrameSize(frameSize(frame), { pixelAspectRatio.num, pixelAspectRatio.den }))

++ {

++     if (frame->hw_frames_ctx) {

++         m_hwFrame = std::move(frame);

++         m_pixelFormat = toQtPixelFormat(HWAccel::format(m_hwFrame.get()));

++         return;

++     }

++

++     m_swFrame = std::move(frame);

++     m_pixelFormat = toQtPixelFormat(AVPixelFormat(m_swFrame->format));

++

++     convertSWFrame();

++ }

++

++ QFFmpegVideoBuffer::~QFFmpegVideoBuffer() = default;

++

++ void QFFmpegVideoBuffer::convertSWFrame()

++ {

++     Q_ASSERT(m_swFrame);

++

++     const auto actualAVPixelFormat = AVPixelFormat(m_swFrame->format);

++     const auto targetAVPixelFormat = toAVPixelFormat(m_pixelFormat);

++

++     const QSize actualSize = frameSize(m_swFrame);

++     if (actualAVPixelFormat != targetAVPixelFormat || isFrameFlipped(*m_swFrame)

++         || m_size != actualSize) {

++         Q_ASSERT(toQtPixelFormat(targetAVPixelFormat) == m_pixelFormat);

++         // convert the format into something we can handle

++         SwsContextUPtr scaleContext = createSwsContext(actualSize, actualAVPixelFormat, m_size,

++                                                        targetAVPixelFormat, SWS_BICUBIC);

++

++         auto newFrame = makeAVFrame();

++         newFrame->width = m_size.width();

++         newFrame->height = m_size.height();

++         newFrame->format = targetAVPixelFormat;

++         const int status = av_frame_get_buffer(newFrame.get(), 0);

++         if (status < 0) {

++             qCWarning(qLcFFmpegVideoBuffer)

++                     << "Failed to allocate frame buffer:" << AVError(status);

++             return;

++         }

++

++         sws_scale(scaleContext.get(), m_swFrame->data, m_swFrame->linesize, 0, m_swFrame->height,

++                   newFrame->data, newFrame->linesize);

++         if (m_frame == m_swFrame.get())

++             m_frame = newFrame.get();

++         m_swFrame = std::move(newFrame);

++     }

++ }

++

++ void QFFmpegVideoBuffer::initTextureConverter(QRhi &rhi)

++ {

++     if (!m_hwFrame)

++         return;

++

++     // don't use the result reference here

++     ensureTextureConverter(rhi);

++ }

++

++ QFFmpeg::TextureConverter &QFFmpegVideoBuffer::ensureTextureConverter(QRhi &rhi)

++ {

++     Q_ASSERT(m_hwFrame);

++

++     HwFrameContextData &frameContextData = HwFrameContextData::ensure(*m_hwFrame);

++     TextureConverter *converter = frameContextData.textureConverterMapper.get(&rhi);

++

++     if (!converter) {

++         bool added = false;

++         std::tie(converter, added) =

++                 frameContextData.textureConverterMapper.tryMap(rhi, TextureConverter(rhi));

++         // no issues are expected if it's already added in another thread, however,it's worth to

++         // check it

++         Q_ASSERT(converter && added);

++     }

++

++     return *converter;

++ }

++

++ QRhi *QFFmpegVideoBuffer::associatedCurrentThreadRhi() const

++ {

++     if (!m_hwFrame)

++         return nullptr;

++

++     HwFrameContextData &frameContextData = HwFrameContextData::ensure(*m_hwFrame);

++     return frameContextData.textureConverterMapper.findRhi(

++             [](QRhi &rhi) { return rhi.thread()->isCurrentThread(); });

++ }

++

++ QVideoFrameFormat::ColorSpace QFFmpegVideoBuffer::colorSpace() const

++ {

++     return fromAvColorSpace(m_frame->colorspace);

++ }

++

++ QVideoFrameFormat::ColorTransfer QFFmpegVideoBuffer::colorTransfer() const

++ {

++     return fromAvColorTransfer(m_frame->color_trc);

++ }

++

++ QVideoFrameFormat::ColorRange QFFmpegVideoBuffer::colorRange() const

++ {

++     return fromAvColorRange(m_frame->color_range);

++ }

++

++ float QFFmpegVideoBuffer::maxNits()

++ {

++     float maxNits = -1;

++     for (int i = 0; i < m_frame->nb_side_data; ++i) {

++         AVFrameSideData *sd = m_frame->side_data[i];

++         // TODO: Longer term we might want to also support HDR10+ dynamic metadata

++         if (sd->type == AV_FRAME_DATA_MASTERING_DISPLAY_METADATA) {

++             auto *data = reinterpret_cast<AVMasteringDisplayMetadata *>(sd->data);

++             // max_luminance is a rational in cd/m2 (nits) already, valid with has_luminance

++             if (!data->has_luminance)

++                 continue;

++             auto maybeLum = QFFmpeg::toReal(data->max_luminance);

++             if (maybeLum)

++                 maxNits = float(*maybeLum);

++         }

++     }

++     return maxNits;

++ }

++

++ QAbstractVideoBuffer::MapData QFFmpegVideoBuffer::map(QVideoFrame::MapMode mode)

++ {

++     if (!m_swFrame) {

++         Q_ASSERT(m_hwFrame && m_hwFrame->hw_frames_ctx);

++         m_swFrame = makeAVFrame();

++         /* retrieve data from GPU to CPU */

++         int ret = av_hwframe_transfer_data(m_swFrame.get(), m_hwFrame.get(), 0);

++         if (ret < 0) {

++             qWarning() << "Error transferring the data to system memory:" << ret;

++             return {};

++         }

++         convertSWFrame();

++     }

++

++     m_mode = mode;

++

++     MapData mapData;

++     auto *desc = QVideoTextureHelper::textureDescription(pixelFormat());

++     mapData.planeCount = desc->nplanes;

++     for (int i = 0; i < mapData.planeCount; ++i) {

++         Q_ASSERT(m_swFrame->linesize[i] >= 0);

++

++         mapData.data[i] = m_swFrame->data[i];

++         mapData.bytesPerLine[i] = m_swFrame->linesize[i];

++         mapData.dataSize[i] = mapData.bytesPerLine[i]*desc->heightForPlane(m_swFrame->height, i);

++     }

++

++     if ((mode & QVideoFrame::WriteOnly) != 0 && m_hwFrame)

++         m_hwFrame.reset();

++

++     return mapData;

++ }

++

++ void QFFmpegVideoBuffer::unmap()

++ {

++     // nothing to do here for SW buffers.

++     // Set NotMapped mode to ensure map/unmap/mapMode consisteny.

++     m_mode = QVideoFrame::NotMapped;

++ }

++

++ QVideoFrameTexturesUPtr QFFmpegVideoBuffer::mapTextures(QRhi &rhi, QVideoFrameTexturesUPtr& oldTextures)

++ {

++     Q_ASSERT(rhi.thread()->isCurrentThread());

++

++     return createTexturesFromHwFrame(rhi, oldTextures);

++ }

++

++ QVideoFrameTexturesUPtr QFFmpegVideoBuffer::createTexturesFromHwFrame(QRhi &rhi, QVideoFrameTexturesUPtr& oldTextures) {

++

++     if (!m_hwFrame)

++         return {};

++

++     // QTBUG-132200:

++     // We aim to set initTextureConverterForAnyRhi=true for as much platforms as we can,

++     // and remove the check after all platforms work fine on CI. If the flag is enabled,

++     // QVideoFrame::toImage can work faster, and we can test hw texture conversion on CI.

++     // Currently, enabling the flag fails some CI platforms.

++     constexpr bool initTextureConverterForAnyRhi = false;

++

++     TextureConverter *converter = initTextureConverterForAnyRhi

++             ? &ensureTextureConverter(rhi)

++             : HwFrameContextData::ensure(*m_hwFrame).textureConverterMapper.get(&rhi);

++

++     if (!converter)

++         return {};

++

++     if (!converter->init(*m_hwFrame))

++         return {};

++

++     const QVideoFrameTextures *oldTexturesRaw = oldTextures.get();

++     if (QVideoFrameTexturesUPtr newTextures = converter->createTextures(*m_hwFrame, oldTextures))

++         return newTextures;

++

++     Q_ASSERT(oldTextures.get() == oldTexturesRaw);

++

++     QVideoFrameTexturesHandlesUPtr oldTextureHandles =

++             oldTextures ? oldTextures->takeHandles() : nullptr;

++     QVideoFrameTexturesHandlesUPtr newTextureHandles =

++             converter->createTextureHandles(*m_hwFrame, std::move(oldTextureHandles));

++

++     if (newTextureHandles) {

++         QVideoFrameTexturesUPtr newTextures = QVideoTextureHelper::createTexturesFromHandles(

++                 std::move(newTextureHandles), rhi, m_pixelFormat, frameSize(m_hwFrame));

++

++         return newTextures;

++     }

++

++     static thread_local int lastFormat = 0;

++     if (std::exchange(lastFormat, m_hwFrame->format) != m_hwFrame->format) // prevent logging spam

++         qWarning() << "    failed to get textures for frame; format:" << m_hwFrame->format;

++

++     return {};

++ }

++

++ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::pixelFormat() const

++ {

++     return m_pixelFormat;

++ }

++

++ QSize QFFmpegVideoBuffer::size() const

++ {

++     return m_size;

++ }

++

++ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat avPixelFormat, bool *needsConversion)

++ {

++     if (needsConversion)

++         *needsConversion = false;

++

++     switch (avPixelFormat) {

++     default:

++         break;

++     case AV_PIX_FMT_NONE:

++         Q_ASSERT(!"Invalid avPixelFormat!");

++         return QVideoFrameFormat::Format_Invalid;

++     case AV_PIX_FMT_ARGB:

++         return QVideoFrameFormat::Format_ARGB8888;

++     case AV_PIX_FMT_0RGB:

++         return QVideoFrameFormat::Format_XRGB8888;

++     case AV_PIX_FMT_BGRA:

++         return QVideoFrameFormat::Format_BGRA8888;

++     case AV_PIX_FMT_BGR0:

++         return QVideoFrameFormat::Format_BGRX8888;

++     case AV_PIX_FMT_ABGR:

++         return QVideoFrameFormat::Format_ABGR8888;

++     case AV_PIX_FMT_0BGR:

++         return QVideoFrameFormat::Format_XBGR8888;

++     case AV_PIX_FMT_RGBA:

++         return QVideoFrameFormat::Format_RGBA8888;

++     case AV_PIX_FMT_RGB0:

++         return QVideoFrameFormat::Format_RGBX8888;

++

++     case AV_PIX_FMT_YUV422P:

++         return QVideoFrameFormat::Format_YUV422P;

++     case AV_PIX_FMT_YUV420P:

++         return QVideoFrameFormat::Format_YUV420P;

++     case AV_PIX_FMT_YUV420P10:

++         return QVideoFrameFormat::Format_YUV420P10;

++     case AV_PIX_FMT_UYVY422:

++         return QVideoFrameFormat::Format_UYVY;

++     case AV_PIX_FMT_YUYV422:

++         return QVideoFrameFormat::Format_YUYV;

++     case AV_PIX_FMT_NV12:

++         return QVideoFrameFormat::Format_NV12;

++     case AV_PIX_FMT_NV21:

++         return QVideoFrameFormat::Format_NV21;

++     case AV_PIX_FMT_GRAY8:

++         return QVideoFrameFormat::Format_Y8;

++     case AV_PIX_FMT_GRAY16:

++         return QVideoFrameFormat::Format_Y16;

++

++     case AV_PIX_FMT_P010:

++         return QVideoFrameFormat::Format_P010;

++     case AV_PIX_FMT_P016:

++         return QVideoFrameFormat::Format_P016;

++     case AV_PIX_FMT_MEDIACODEC:

++         return QVideoFrameFormat::Format_SamplerExternalOES;

++     }

++

++     if (needsConversion)

++         *needsConversion = true;

++

++     const AVPixFmtDescriptor *descriptor = av_pix_fmt_desc_get(avPixelFormat);

++

++     if (descriptor->flags & AV_PIX_FMT_FLAG_RGB)

++         return QVideoFrameFormat::Format_RGBA8888;

++

++     if (descriptor->comp[0].depth > 8)

++         return QVideoFrameFormat::Format_P016;

++     return QVideoFrameFormat::Format_YUV420P;

++ }

++

++ AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat)

++ {

++     switch (pixelFormat) {

++     default:

++     case QVideoFrameFormat::Format_Invalid:

++     case QVideoFrameFormat::Format_AYUV:

++     case QVideoFrameFormat::Format_AYUV_Premultiplied:

++     case QVideoFrameFormat::Format_YV12:

++     case QVideoFrameFormat::Format_IMC1:

++     case QVideoFrameFormat::Format_IMC2:

++     case QVideoFrameFormat::Format_IMC3:

++     case QVideoFrameFormat::Format_IMC4:

++         return AV_PIX_FMT_NONE;

++     case QVideoFrameFormat::Format_Jpeg:

++         // We're using the data from the converted QImage here, which is in BGRA.

++         return AV_PIX_FMT_BGRA;

++     case QVideoFrameFormat::Format_ARGB8888:

++         return AV_PIX_FMT_ARGB;

++     case QVideoFrameFormat::Format_ARGB8888_Premultiplied:

++     case QVideoFrameFormat::Format_XRGB8888:

++         return AV_PIX_FMT_0RGB;

++     case QVideoFrameFormat::Format_BGRA8888:

++         return AV_PIX_FMT_BGRA;

++     case QVideoFrameFormat::Format_BGRA8888_Premultiplied:

++     case QVideoFrameFormat::Format_BGRX8888:

++         return AV_PIX_FMT_BGR0;

++     case QVideoFrameFormat::Format_ABGR8888:

++         return AV_PIX_FMT_ABGR;

++     case QVideoFrameFormat::Format_XBGR8888:

++         return AV_PIX_FMT_0BGR;

++     case QVideoFrameFormat::Format_RGBA8888:

++         return AV_PIX_FMT_RGBA;

++     // to be added in 6.8:

++     // case QVideoFrameFormat::Format_RGBA8888_Premultiplied:

++     case QVideoFrameFormat::Format_RGBX8888:

++         return AV_PIX_FMT_RGB0;

++

++     case QVideoFrameFormat::Format_YUV422P:

++         return AV_PIX_FMT_YUV422P;

++     case QVideoFrameFormat::Format_YUV420P:

++         return AV_PIX_FMT_YUV420P;

++     case QVideoFrameFormat::Format_YUV420P10:

++         return AV_PIX_FMT_YUV420P10;

++     case QVideoFrameFormat::Format_UYVY:

++         return AV_PIX_FMT_UYVY422;

++     case QVideoFrameFormat::Format_YUYV:

++         return AV_PIX_FMT_YUYV422;

++     case QVideoFrameFormat::Format_NV12:

++         return AV_PIX_FMT_NV12;

++     case QVideoFrameFormat::Format_NV21:

++         return AV_PIX_FMT_NV21;

++     case QVideoFrameFormat::Format_Y8:

++         return AV_PIX_FMT_GRAY8;

++     case QVideoFrameFormat::Format_Y16:

++         return AV_PIX_FMT_GRAY16;

++

++     case QVideoFrameFormat::Format_P010:

++         return AV_PIX_FMT_P010;

++     case QVideoFrameFormat::Format_P016:

++         return AV_PIX_FMT_P016;

++

++     case QVideoFrameFormat::Format_SamplerExternalOES:

++         return AV_PIX_FMT_MEDIACODEC;

++     }

++ }

++

++ QT_END_NAMESPACE

Revision control