Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpeg_p.h"
#include <QtCore/qdebug.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qscopeguard.h>
extern "C" {
#include <libavutil/pixdesc.h>
#include <libavutil/samplefmt.h>
#include <libavutil/error.h>
#ifdef Q_OS_DARWIN
#include <libavutil/hwcontext_videotoolbox.h>
#endif
}
QT_BEGIN_NAMESPACE
Q_STATIC_LOGGING_CATEGORY(qLcFFmpegUtils, "qt.multimedia.ffmpeg.utils");
namespace QFFmpeg {
namespace {
bool isAVFormatSupported(const Codec &codec, AVPixelFormat format)
{
if (codec.type() == AVMEDIA_TYPE_VIDEO) {
auto checkFormat = [format](AVPixelFormat f) {
return f == format;
};
return findAVPixelFormat(codec, checkFormat).has_value();
}
return false;
}
bool isAVFormatSupported(const Codec &codec, AVSampleFormat format)
{
if (codec.type() == AVMEDIA_TYPE_AUDIO) {
const auto sampleFormats = codec.sampleFormats();
return ranges::contains(sampleFormats, format);
}
return false;
}
} // namespace
bool isAVFormatSupported(const Codec &codec, const PixelOrSampleFormat &format)
{
return std::visit([&](auto fmt) {
return isAVFormatSupported(codec, fmt);
}, format);
}
bool isHwPixelFormat(AVPixelFormat format)
{
const auto desc = av_pix_fmt_desc_get(format);
return desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) != 0;
}
void applyExperimentalCodecOptions(const Codec &codec, AVDictionary **opts)
{
if (codec.isExperimental()) {
qCWarning(qLcFFmpegUtils) << "Applying the option 'strict -2' for the experimental codec"
<< codec.name() << ". it's unlikely to work properly";
av_dict_set(opts, "strict", "-2", 0);
}
}
AVPixelFormat pixelFormatForHwDevice(AVHWDeviceType deviceType)
{
switch (deviceType) {
case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
return AV_PIX_FMT_VIDEOTOOLBOX;
case AV_HWDEVICE_TYPE_VAAPI:
return AV_PIX_FMT_VAAPI;
case AV_HWDEVICE_TYPE_MEDIACODEC:
return AV_PIX_FMT_MEDIACODEC;
case AV_HWDEVICE_TYPE_CUDA:
return AV_PIX_FMT_CUDA;
case AV_HWDEVICE_TYPE_VDPAU:
return AV_PIX_FMT_VDPAU;
case AV_HWDEVICE_TYPE_OPENCL:
return AV_PIX_FMT_OPENCL;
case AV_HWDEVICE_TYPE_QSV:
return AV_PIX_FMT_QSV;
case AV_HWDEVICE_TYPE_D3D11VA:
return AV_PIX_FMT_D3D11;
#if QT_FFMPEG_HAS_D3D12VA
case AV_HWDEVICE_TYPE_D3D12VA:
return AV_PIX_FMT_D3D12;
#endif
case AV_HWDEVICE_TYPE_DXVA2:
return AV_PIX_FMT_DXVA2_VLD;
case AV_HWDEVICE_TYPE_DRM:
return AV_PIX_FMT_DRM_PRIME;
case AV_HWDEVICE_TYPE_VULKAN:
return AV_PIX_FMT_VULKAN;
default:
return AV_PIX_FMT_NONE;
}
}
AVPacketSideData *addStreamSideData(AVStream *stream, AVPacketSideData sideData)
{
QScopeGuard freeData([&sideData]() { av_free(sideData.data); });
#if QT_FFMPEG_STREAM_SIDE_DATA_DEPRECATED
AVPacketSideData *result = av_packet_side_data_add(
&stream->codecpar->coded_side_data,
&stream->codecpar->nb_coded_side_data,
sideData.type,
sideData.data,
sideData.size,
0);
if (result) {
// If the result is not null, the ownership is taken by AVStream,
// otherwise the data must be deleted.
freeData.dismiss();
return result;
}
#else
Q_UNUSED(stream);
// TODO: implement for older FFmpeg versions
qWarning() << "Adding stream side data is not supported for FFmpeg < 6.1";
#endif
return nullptr;
}
const AVPacketSideData *streamSideData(const AVStream *stream, AVPacketSideDataType type)
{
Q_ASSERT(stream);
#if QT_FFMPEG_STREAM_SIDE_DATA_DEPRECATED
return av_packet_side_data_get(stream->codecpar->coded_side_data,
stream->codecpar->nb_coded_side_data, type);
#else
const auto end = stream->side_data + stream->nb_side_data;
const auto found = std::find_if(stream->side_data, end, [&](const auto &item) {
return item.type == type;
});
return found == end ? nullptr : found;
#endif
}
SwrContextUPtr createResampleContext(const AVAudioFormat &inputFormat,
const AVAudioFormat &outputFormat)
{
SwrContext *resampler = nullptr;
#if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
#if QT_FFMPEG_SWR_CONST_CH_LAYOUT
using AVChannelLayoutPrm = const AVChannelLayout*;
#else
using AVChannelLayoutPrm = AVChannelLayout*;
#endif
swr_alloc_set_opts2(&resampler,
const_cast<AVChannelLayoutPrm>(&outputFormat.channelLayout),
outputFormat.sampleFormat,
outputFormat.sampleRate,
const_cast<AVChannelLayoutPrm>(&inputFormat.channelLayout),
inputFormat.sampleFormat,
inputFormat.sampleRate,
0,
nullptr);
#else
resampler = swr_alloc_set_opts(nullptr,
outputFormat.channelLayoutMask,
outputFormat.sampleFormat,
outputFormat.sampleRate,
inputFormat.channelLayoutMask,
inputFormat.sampleFormat,
inputFormat.sampleRate,
0,
nullptr);
#endif
auto error = QFFmpeg::AVError{
swr_init(resampler),
};
if (error != QFFmpeg::AVError::Success) {
qCWarning(qLcFFmpegUtils) << "Failed to initialize audio resampler:" << error;
return nullptr;
}
return SwrContextUPtr(resampler);
}
QVideoFrameFormat::ColorTransfer fromAvColorTransfer(AVColorTransferCharacteristic colorTrc) {
switch (colorTrc) {
case AVCOL_TRC_BT709:
// The following three cases have transfer characteristics identical to BT709
case AVCOL_TRC_BT1361_ECG:
case AVCOL_TRC_BT2020_10:
case AVCOL_TRC_BT2020_12:
case AVCOL_TRC_SMPTE240M: // almost identical to bt709
return QVideoFrameFormat::ColorTransfer_BT709;
case AVCOL_TRC_GAMMA22:
case AVCOL_TRC_SMPTE428: // No idea, let's hope for the best...
case AVCOL_TRC_IEC61966_2_1: // sRGB, close enough to 2.2...
case AVCOL_TRC_IEC61966_2_4: // not quite, but probably close enough
return QVideoFrameFormat::ColorTransfer_Gamma22;
case AVCOL_TRC_GAMMA28:
return QVideoFrameFormat::ColorTransfer_Gamma28;
case AVCOL_TRC_SMPTE170M:
return QVideoFrameFormat::ColorTransfer_BT601;
case AVCOL_TRC_LINEAR:
return QVideoFrameFormat::ColorTransfer_Linear;
case AVCOL_TRC_SMPTE2084:
return QVideoFrameFormat::ColorTransfer_ST2084;
case AVCOL_TRC_ARIB_STD_B67:
return QVideoFrameFormat::ColorTransfer_STD_B67;
default:
break;
}
return QVideoFrameFormat::ColorTransfer_Unknown;
}
AVColorTransferCharacteristic toAvColorTransfer(QVideoFrameFormat::ColorTransfer colorTrc)
{
switch (colorTrc) {
case QVideoFrameFormat::ColorTransfer_BT709:
return AVCOL_TRC_BT709;
case QVideoFrameFormat::ColorTransfer_BT601:
return AVCOL_TRC_BT709; // which one is the best?
case QVideoFrameFormat::ColorTransfer_Linear:
return AVCOL_TRC_SMPTE2084;
case QVideoFrameFormat::ColorTransfer_Gamma22:
return AVCOL_TRC_GAMMA22;
case QVideoFrameFormat::ColorTransfer_Gamma28:
return AVCOL_TRC_GAMMA28;
case QVideoFrameFormat::ColorTransfer_ST2084:
return AVCOL_TRC_SMPTE2084;
case QVideoFrameFormat::ColorTransfer_STD_B67:
return AVCOL_TRC_ARIB_STD_B67;
default:
return AVCOL_TRC_UNSPECIFIED;
}
}
QVideoFrameFormat::ColorSpace fromAvColorSpace(AVColorSpace colorSpace)
{
switch (colorSpace) {
default:
case AVCOL_SPC_UNSPECIFIED:
case AVCOL_SPC_RESERVED:
case AVCOL_SPC_FCC:
case AVCOL_SPC_SMPTE240M:
case AVCOL_SPC_YCGCO:
case AVCOL_SPC_SMPTE2085:
case AVCOL_SPC_CHROMA_DERIVED_NCL:
case AVCOL_SPC_CHROMA_DERIVED_CL:
case AVCOL_SPC_ICTCP: // BT.2100 ICtCp
return QVideoFrameFormat::ColorSpace_Undefined;
case AVCOL_SPC_RGB:
return QVideoFrameFormat::ColorSpace_AdobeRgb;
case AVCOL_SPC_BT709:
return QVideoFrameFormat::ColorSpace_BT709;
case AVCOL_SPC_BT470BG: // BT601
case AVCOL_SPC_SMPTE170M: // Also BT601
return QVideoFrameFormat::ColorSpace_BT601;
case AVCOL_SPC_BT2020_NCL: // Non constant luminence
case AVCOL_SPC_BT2020_CL: // Constant luminence
return QVideoFrameFormat::ColorSpace_BT2020;
}
}
AVColorSpace toAvColorSpace(QVideoFrameFormat::ColorSpace colorSpace)
{
switch (colorSpace) {
case QVideoFrameFormat::ColorSpace_BT601:
return AVCOL_SPC_BT470BG;
case QVideoFrameFormat::ColorSpace_BT709:
return AVCOL_SPC_BT709;
case QVideoFrameFormat::ColorSpace_AdobeRgb:
return AVCOL_SPC_RGB;
case QVideoFrameFormat::ColorSpace_BT2020:
return AVCOL_SPC_BT2020_CL;
default:
return AVCOL_SPC_UNSPECIFIED;
}
}
QVideoFrameFormat::ColorRange fromAvColorRange(AVColorRange colorRange)
{
switch (colorRange) {
case AVCOL_RANGE_MPEG:
return QVideoFrameFormat::ColorRange_Video;
case AVCOL_RANGE_JPEG:
return QVideoFrameFormat::ColorRange_Full;
default:
return QVideoFrameFormat::ColorRange_Unknown;
}
}
AVColorRange toAvColorRange(QVideoFrameFormat::ColorRange colorRange)
{
switch (colorRange) {
case QVideoFrameFormat::ColorRange_Video:
return AVCOL_RANGE_MPEG;
case QVideoFrameFormat::ColorRange_Full:
return AVCOL_RANGE_JPEG;
default:
return AVCOL_RANGE_UNSPECIFIED;
}
}
AVHWDeviceContext* avFrameDeviceContext(const AVFrame* frame) {
if (!frame)
return {};
if (!frame->hw_frames_ctx)
return {};
const auto *frameCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
if (!frameCtx)
return {};
return frameCtx->device_ctx;
}
SwsContextUPtr createSwsContext(const QSize &srcSize, AVPixelFormat srcPixFmt, const QSize &dstSize,
AVPixelFormat dstPixFmt, SwsFlags conversionType)
{
SwsContext *result =
sws_getContext(srcSize.width(), srcSize.height(), srcPixFmt, dstSize.width(),
dstSize.height(), dstPixFmt, conversionType, nullptr, nullptr, nullptr);
if (!result)
qCWarning(qLcFFmpegUtils) << "Cannot create sws context for:\n"
<< "srcSize:" << srcSize
<< "srcPixFmt:" << srcPixFmt
<< "dstSize:" << dstSize
<< "dstPixFmt:" << dstPixFmt
<< "conversionType:" << conversionType;
return SwsContextUPtr(result);
}
void freeAVIOContext(AVIOContext *&ctx)
{
if (ctx == nullptr)
return;
av_freep(&ctx->buffer);
avio_context_free(&ctx);
Q_ASSERT(ctx == nullptr);
}
QSize frameSize(const AVFrame &frame)
{
return QSize(frame.width, frame.height);
}
QSize frameSize(const AVFrame *frame)
{
return frame ? frameSize(*frame) : QSize();
}
QSize frameSize(const AVFrameUPtr &frame)
{
return frame ? frameSize(*frame) : QSize();
}
#ifdef Q_OS_DARWIN
bool isCVFormatSupported(uint32_t cvFormat)
{
return av_map_videotoolbox_format_to_pixfmt(cvFormat) != AV_PIX_FMT_NONE;
}
std::string cvFormatToString(uint32_t cvFormat)
{
auto formatDescIt = std::make_reverse_iterator(reinterpret_cast<const char *>(&cvFormat));
return std::string(formatDescIt - 4, formatDescIt);
}
#endif
} // namespace QFFmpeg
QDebug operator<<(QDebug dbg, const AVRational &value)
{
dbg << value.num << "/" << value.den;
return dbg;
}
QDebug operator<<(QDebug dbg, const AVDictionary &dict)
{
char *buffer = 0;
auto freeBuffer = QScopeGuard([&] {
av_free(buffer);
});
int status = av_dict_get_string(&dict, &buffer, '=', ',');
if (status < 0 || !buffer)
return dbg << "Failed to print AVDictionary";
dbg << buffer;
return dbg;
}
QDebug operator<<(QDebug dbg, const QFFmpeg::AVDictionaryHolder &dict)
{
const AVDictionary *rawDict = dict.opts;
if (rawDict)
return dbg << *rawDict;
else
return dbg << "Empty AVDictionaryHolder";
}
QDebug operator<<(QDebug dbg, QFFmpeg::AVError error)
{
if (error == QFFmpeg::AVError::Success) {
dbg << "Success";
return dbg;
}
char errBuf[AV_ERROR_MAX_STRING_SIZE];
dbg << av_make_error_string(errBuf, AV_ERROR_MAX_STRING_SIZE, qToUnderlying(error));
return dbg;
}
QDebug operator<<(QDebug dbg, AVPixelFormat fmt)
{
const char *str = av_get_pix_fmt_name(fmt);
if (str)
dbg << str;
else
dbg << "<unknown pixel format>";
return dbg;
}
QDebug operator<<(QDebug dbg, AVHWDeviceType type)
{
const char *str = av_hwdevice_get_type_name(type);
if (str)
dbg << str;
else
dbg << "<unknown hw device type>";
return dbg;
}
QDebug operator<<(QDebug dbg, AVMediaType type)
{
const char *str = av_get_media_type_string(type);
if (str)
dbg << str;
else
dbg << "<unknown hw media type>";
return dbg;
}
QDebug operator<<(QDebug dbg, AVCodecID id)
{
const char *str = avcodec_get_name(id);
if (str)
dbg << str;
else
dbg << "<unknown codec id>";
return dbg;
}
QDebug operator<<(QDebug dbg, const AVOutputFormat *fmt)
{
if (fmt == nullptr) {
dbg << "<null>";
return dbg;
}
const char *str = fmt->name;
if (str)
dbg << str;
else
dbg << "<unknown output format>";
return dbg;
}
QDebug operator<<(QDebug dbg, const AVInputFormat *fmt)
{
if (fmt == nullptr) {
dbg << "<null>";
return dbg;
}
const char *str = fmt->name;
if (str)
dbg << str;
else
dbg << "<unknown input format>";
return dbg;
}
#if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
QDebug operator<<(QDebug dbg, const AVChannelLayout &layout)
{
dbg << '[';
dbg << "nb_channels:" << layout.nb_channels;
dbg << ", order:" << layout.order;
if (layout.order == AV_CHANNEL_ORDER_NATIVE || layout.order == AV_CHANNEL_ORDER_AMBISONIC)
dbg << ", mask:" << Qt::bin << layout.u.mask << Qt::dec;
else if (layout.order == AV_CHANNEL_ORDER_CUSTOM && layout.u.map)
dbg << ", id: " << layout.u.map->id;
dbg << ']';
return dbg;
}
#endif
#if QT_FFMPEG_HAS_AVCODEC_GET_SUPPORTED_CONFIG
QDebug operator<<(QDebug dbg, const AVCodecConfig value)
{
switch (value) {
case AV_CODEC_CONFIG_CHANNEL_LAYOUT:
dbg << "AV_CODEC_CONFIG_CHANNEL_LAYOUT";
break;
case AV_CODEC_CONFIG_COLOR_RANGE:
dbg << "AV_CODEC_CONFIG_COLOR_RANGE";
break;
case AV_CODEC_CONFIG_COLOR_SPACE:
dbg << "AV_CODEC_CONFIG_COLOR_SPACE";
break;
case AV_CODEC_CONFIG_FRAME_RATE:
dbg << "AV_CODEC_CONFIG_FRAME_RATE";
break;
case AV_CODEC_CONFIG_PIX_FORMAT:
dbg << "AV_CODEC_CONFIG_PIX_FORMAT";
break;
case AV_CODEC_CONFIG_SAMPLE_FORMAT:
dbg << "AV_CODEC_CONFIG_SAMPLE_FORMAT";
break;
case AV_CODEC_CONFIG_SAMPLE_RATE:
dbg << "AV_CODEC_CONFIG_SAMPLE_RATE";
break;
default:
dbg << "<UNKNOWN_CODEC_CONFIG>";
break;
}
return dbg;
}
#endif
namespace {
struct FlagName {
int value;
const char *name;
};
QString flagsToString(int flags, const QSpan<const FlagName> &flagNames)
{
QString result;
int leftover = flags;
for (const auto &flagAndName : flagNames)
if ((flags & flagAndName.value) != 0) {
leftover &= ~flagAndName.value;
if (!result.isEmpty())
result += u", ";
result += QLatin1StringView(flagAndName.name);
}
if (leftover) {
if (!result.isEmpty())
result += u", ";
result += QString::number(leftover, 16);
}
return result;
}
} // namespace
QString QFFmpeg::toString(AVCodecCapabilities capabilities)
{
static constexpr std::array<FlagName, 21> flagNames = {{
{ AV_CODEC_CAP_DRAW_HORIZ_BAND, "DRAW_HORIZ_BAND" },
{ AV_CODEC_CAP_DR1, "DR1" },
{ AV_CODEC_CAP_DELAY, "DELAY" },
{ AV_CODEC_CAP_SMALL_LAST_FRAME, "SMALL_LAST_FRAME" },
#ifdef AV_CODEC_CAP_SUBFRAMES
{ AV_CODEC_CAP_SUBFRAMES, "SUBFRAMES" },
#endif
{ AV_CODEC_CAP_EXPERIMENTAL, "EXPERIMENTAL" },
{ AV_CODEC_CAP_CHANNEL_CONF, "CHANNEL_CONF" },
{ AV_CODEC_CAP_FRAME_THREADS, "FRAME_THREADS" },
{ AV_CODEC_CAP_SLICE_THREADS, "SLICE_THREADS" },
{ AV_CODEC_CAP_PARAM_CHANGE, "PARAM_CHANGE" },
{ AV_CODEC_CAP_OTHER_THREADS, "OTHER_THREADS" },
{ AV_CODEC_CAP_VARIABLE_FRAME_SIZE, "VARIABLE_FRAME_SIZE" },
{ AV_CODEC_CAP_AVOID_PROBING, "AVOID_PROBING" },
{ AV_CODEC_CAP_HARDWARE, "HARDWARE" },
{ AV_CODEC_CAP_HYBRID, "HYBRID" },
{ AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE, "ENCODER_REORDERED_OPAQUE" },
{ AV_CODEC_CAP_ENCODER_FLUSH, "ENCODER_FLUSH" },
}};
return flagsToString(qToUnderlying(capabilities), QSpan(flagNames));
}
QString QFFmpeg::toString(AVPixelFormatFlags flags)
{
static constexpr std::array<FlagName, 9> flagNames = {{
{ AV_PIX_FMT_FLAG_BE, "BE" },
{ AV_PIX_FMT_FLAG_PAL, "PAL" },
{ AV_PIX_FMT_FLAG_BITSTREAM, "BITSTREAM" },
{ AV_PIX_FMT_FLAG_HWACCEL, "HWACCEL" },
{ AV_PIX_FMT_FLAG_PLANAR, "PLANAR" },
{ AV_PIX_FMT_FLAG_RGB, "RGB" },
{ AV_PIX_FMT_FLAG_ALPHA, "ALPHA" },
{ AV_PIX_FMT_FLAG_BAYER, "BAYER" },
{ AV_PIX_FMT_FLAG_FLOAT, "FLOAT" },
}};
return flagsToString(qToUnderlying(flags), QSpan(flagNames));
}
QString QFFmpeg::toString(AVHwConfigMethods methods)
{
static constexpr std::array<FlagName, 4> flagNames = {{
{ AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX, "HW_DEVICE_CTX" },
{ AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX, "HW_FRAMES_CTX" },
{ AV_CODEC_HW_CONFIG_METHOD_INTERNAL, "INTERNAL" },
{ AV_CODEC_HW_CONFIG_METHOD_AD_HOC, "AD_HOC" }
}};
return flagsToString(qToUnderlying(methods), QSpan(flagNames));
}
QDebug operator<<(QDebug dbg, QFFmpeg::AVCodecCapabilities caps)
{
return dbg << QFFmpeg::toString(caps);
}
QDebug operator<<(QDebug dbg, QFFmpeg::AVPixelFormatFlags flags)
{
return dbg << QFFmpeg::toString(flags);
}
QDebug operator<<(QDebug dbg, QFFmpeg::AVHwConfigMethods methods)
{
return dbg << QFFmpeg::toString(methods);
}
QT_END_NAMESPACE