qffmpegcodecstorage.cpp

++ // Copyright (C) 2024 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++

++ #include "qffmpegcodecstorage_p.h"

++

++ #include "qffmpeg_p.h"

++ #include "qffmpeghwaccel_p.h"

++

++ #include <QtCore/qapplicationstatic.h>

++ #include <QtCore/qoperatingsystemversion.h>

++ #include <QtCore/qdebug.h>

++ #include <QtCore/qloggingcategory.h>

++ #include <QtCore/qspan.h>

++

++ #include <algorithm>

++ #include <array>

++ #include <future>

++ #include <set>

++ #include <string>

++ #include <unordered_set>

++ #include <vector>

++

++ extern "C" {

++ #include <libavcodec/avcodec.h>

++ #include <libavutil/pixdesc.h>

++ #include <libavutil/samplefmt.h>

++ }

++

++ #ifdef Q_OS_ANDROID

++ #  include <QtCore/qjniobject.h>

++ #  include <QtCore/qjniarray.h>

++ #  include <QtCore/qjnitypes.h>

++

++ #  include <QtFFmpegMediaPluginImpl/private/qandroidvideojnitypes_p.h>

++ #endif

++

++ QT_BEGIN_NAMESPACE

++

++ Q_STATIC_LOGGING_CATEGORY(qLcCodecStorage, "qt.multimedia.ffmpeg.codecstorage");

++

++ namespace QFFmpeg {

++

++ namespace ranges = QtMultimediaPrivate::ranges;

++ using namespace Qt::Literals;

++

++ namespace {

++

++ using CodecsStorage = std::vector<Codec>;

++

++ struct CodecsComparator

++ {

++     bool operator()(const Codec &a, const Codec &b) const

++     {

++         return a.id() < b.id() || (a.id() == b.id() && a.isExperimental() < b.isExperimental());

++     }

++

++     bool operator()(const Codec &codec, AVCodecID id) const { return codec.id() < id; }

++     bool operator()(AVCodecID id, const Codec &codec) const { return id < codec.id(); }

++     bool operator()(AVCodecID a, AVCodecID b) const { return a < b; }

++ };

++

++ void dumpCodecInfo(const Codec &codec)

++ {

++     const auto roles =

++             codec.isEncoder() ? codec.isDecoder() ? "encoder/decoder:" : "encoder:" : "decoder:";

++

++     qCDebug(qLcCodecStorage) << codec.type() << roles << codec.name() << "id:" << codec.id()

++                              << "capabilities:" << AVCodecCapabilities(codec.capabilities());

++

++     if (codec.type() == AVMEDIA_TYPE_VIDEO) {

++         const auto pixelFormats = codec.pixelFormats();

++         if (!pixelFormats.empty()) {

++             qCDebug(qLcCodecStorage) << "  pixelFormats:";

++             for (AVPixelFormat f : pixelFormats) {

++                 auto desc = av_pix_fmt_desc_get(f);

++                 qCDebug(qLcCodecStorage)

++                         << "    id:" << f << desc->name << "depth:" << desc->comp[0].depth

++                         << "flags:" << AVPixelFormatFlags(desc->flags);

++             }

++         } else {

++             qCDebug(qLcCodecStorage) << "  pixelFormats: null";

++         }

++     } else if (codec.type() == AVMEDIA_TYPE_AUDIO) {

++         const auto sampleFormats = codec.sampleFormats();

++         if (!sampleFormats.empty()) {

++             qCDebug(qLcCodecStorage) << "  sampleFormats:";

++             for (auto f : sampleFormats) {

++                 const auto name = av_get_sample_fmt_name(f);

++                 qCDebug(qLcCodecStorage) << "    id:" << f << (name ? name : "unknown")

++                                          << "bytes_per_sample:" << av_get_bytes_per_sample(f)

++                                          << "is_planar:" << av_sample_fmt_is_planar(f);

++             }

++         } else {

++             qCDebug(qLcCodecStorage) << "  sampleFormats: null";

++         }

++     }

++

++     const std::vector<const AVCodecHWConfig*> hwConfigs = codec.hwConfigs();

++     if (!hwConfigs.empty()) {

++         qCDebug(qLcCodecStorage) << "  hw config:";

++         for (const AVCodecHWConfig* config : hwConfigs) {

++             const auto pixFmtForDevice = pixelFormatForHwDevice(config->device_type);

++             auto pixFmtDesc = av_pix_fmt_desc_get(config->pix_fmt);

++             auto pixFmtForDeviceDesc = av_pix_fmt_desc_get(pixFmtForDevice);

++             qCDebug(qLcCodecStorage)

++                     << "    device_type:" << config->device_type << "pix_fmt:" << config->pix_fmt

++                     << (pixFmtDesc ? pixFmtDesc->name : "unknown")

++                     << "pixelFormatForHwDevice:" << pixelFormatForHwDevice(config->device_type)

++                     << (pixFmtForDeviceDesc ? pixFmtForDeviceDesc->name : "unknown")

++                     << "hw_config_methods:" << AVHwConfigMethods(config->methods);

++         }

++     }

++ }

++

++ enum class MFCodecCheckResult {

++     supported_mf_codec,

++     unsupported_mf_codec,

++     not_an_mf_codec,

++ };

++

++ MFCodecCheckResult isValidMFEncoder([[maybe_unused]] const Codec &codec)

++ {

++     if constexpr (QOperatingSystemVersion::currentType() == QOperatingSystemVersion::Windows) {

++         if (!codec.name().endsWith("_mf"_L1))

++             return MFCodecCheckResult::not_an_mf_codec;

++

++         AVCodecContextUPtr ctx{ avcodec_alloc_context3(codec.get()) };

++         if (!ctx)

++             return MFCodecCheckResult::unsupported_mf_codec;

++

++         ctx->width = 1280;

++         ctx->height = 720;

++         ctx->time_base = { 1, 30 };

++         ctx->framerate = { 30, 1 };

++         ctx->pix_fmt = AV_PIX_FMT_NV12;

++

++         const int ret = avcodec_open2(ctx.get(), codec.get(), nullptr);

++         if (ret == AVERROR(ENOSYS)) {

++             qCDebug(qLcCodecStorage) << "MF codec" << codec.name() << "is not available.";

++             return MFCodecCheckResult::unsupported_mf_codec;

++         }

++

++         if (ret < 0) {

++             qCDebug(qLcCodecStorage) << "MF codec" << codec.name()

++                                      << "is not supported due to avcodec_open2 failure:" << ret

++                                      << QFFmpeg::AVError(ret);

++             return MFCodecCheckResult::unsupported_mf_codec;

++         }

++

++         return MFCodecCheckResult::supported_mf_codec;

++     } else {

++         return MFCodecCheckResult::not_an_mf_codec;

++     }

++ }

++

++ bool isCodecValid(const Codec &codec, QSpan<const AVHWDeviceType> availableHwDeviceTypes,

++                   const std::optional<std::unordered_set<AVCodecID>> &codecAvailableOnDevice)

++ {

++     if (codec.type() != AVMEDIA_TYPE_VIDEO)

++         return true;

++

++     const bool isHardwareCodec = codec.capabilities() & AV_CODEC_CAP_HARDWARE;

++

++     if (codecAvailableOnDevice && isHardwareCodec) {

++         Q_ASSERT(QOperatingSystemVersion::currentType() == QOperatingSystemVersion::Android);

++         // MediaCodec in Android is used for hardware-accelerated media processing. That is why

++         // before marking it as valid, we need to make sure if it is available on current device.

++         if (codecAvailableOnDevice->count(codec.id()) == 0)

++             return false; // Codec is not in platform's allow-list

++     }

++

++     const auto pixelFormats = codec.pixelFormats();

++     if (pixelFormats.empty()) {

++ #if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)

++         //  Disable V4L2 M2M codecs for encoding for now,

++         //  TODO: Investigate on how to get them working

++         if (codec.name().contains(QLatin1StringView{ "_v4l2m2m" }) && codec.isEncoder())

++             return false;

++ #endif

++

++         return true; // When the codec reports no pixel formats, format support is unknown.

++     }

++

++     if (codec.isEncoder() && isValidMFEncoder(codec) == MFCodecCheckResult::unsupported_mf_codec)

++         return false; // Unsupported Media Foundation codec

++

++     if (!findAVPixelFormat(codec, &isHwPixelFormat))

++         return true; // Codec does not support any hw pixel formats, so no further checks are needed

++

++     if (!isHardwareCodec)

++         return true; // Codec does not support hardware processing, so no further checks are needed

++

++     auto checkDeviceType = [codec](AVHWDeviceType type) {

++         return isAVFormatSupported(codec, pixelFormatForHwDevice(type));

++     };

++

++     return ranges::any_of(availableHwDeviceTypes, checkDeviceType);

++ }

++

++ std::optional<std::unordered_set<AVCodecID>> availableHWCodecs(const CodecRole type)

++ {

++ #ifdef Q_OS_ANDROID

++     using namespace Qt::StringLiterals;

++     using namespace QtJniTypes;

++     std::unordered_set<AVCodecID> availabeCodecs;

++

++     auto getCodecId = [](const QString &codecName) {

++         if (codecName == "3gpp"_L1)

++             return AV_CODEC_ID_H263;

++         if (codecName == "avc"_L1)

++             return AV_CODEC_ID_H264;

++         if (codecName == "hevc"_L1)

++             return AV_CODEC_ID_HEVC;

++         if (codecName == "mp4v-es"_L1)

++             return AV_CODEC_ID_MPEG4;

++         if (codecName == "x-vnd.on2.vp8"_L1)

++             return AV_CODEC_ID_VP8;

++         if (codecName == "x-vnd.on2.vp9"_L1)

++             return AV_CODEC_ID_VP9;

++         return AV_CODEC_ID_NONE;

++     };

++

++     const QJniArray jniCodecs = QtVideoDeviceManager::callStaticMethod<String[]>(

++             type == CodecRole::Encoders ? "getHWVideoEncoders" : "getHWVideoDecoders");

++

++     for (const auto &codec : jniCodecs)

++         availabeCodecs.insert(getCodecId(codec.toString()));

++     return availabeCodecs;

++ #else

++     Q_UNUSED(type);

++     return {};

++ #endif

++ }

++

++ struct CodecStoreSingleton

++ {

++     std::shared_future<std::array<CodecsStorage, 2>> codecStoreFuture;

++

++     static bool isExcludedEncoder(QLatin1String codecName)

++     {

++         static const std::set<std::string, std::less<>> excludeSet = [] {

++             std::set<std::string, std::less<>> s;

++             const QByteArray excludeEnv = qgetenv("QT_FFMPEG_EXCLUDE_ENCODERS");

++             if (excludeEnv.isEmpty())

++                 return s;

++             const QStringList parts = QString::fromUtf8(excludeEnv).split(u',', Qt::SkipEmptyParts);

++             for (const QString &p : parts) {

++                 const QString t = p.trimmed().toLower();

++                 if (!t.isEmpty())

++                     s.insert(t.toStdString());

++             }

++             return s;

++         }();

++

++         std::string_view codecNameView{ codecName.data(), size_t(codecName.size()) };

++

++         if (excludeSet.count(codecNameView)) {

++             qCDebug(qLcCodecStorage)

++                     << "Skip encoder" << codecName << "due to QT_FFMPEG_EXCLUDE_ENCODERS";

++             return true;

++         }

++         return false;

++     }

++

++     static std::array<CodecsStorage, 2> enumerateCodecs()

++     {

++         std::array<CodecsStorage, 2> result;

++         const auto platformHwEncoders = availableHWCodecs(CodecRole::Encoders);

++         const auto platformHwDecoders = availableHWCodecs(CodecRole::Decoders);

++

++         for (const Codec codec : allCodecs) {

++             // TODO: to be investigated

++             // FFmpeg functions avcodec_find_decoder/avcodec_find_encoder

++             // find experimental codecs in the last order,

++             // now we don't consider them at all since they are supposed to

++             // be not stable, maybe we shouldn't.

++             // Currently, it's possible to turn them on for testing purposes.

++

++             static const auto experimentalCodecsEnabled =

++                     qEnvironmentVariableIntValue("QT_ENABLE_EXPERIMENTAL_CODECS");

++

++             if (!experimentalCodecsEnabled && codec.isExperimental()) {

++                 qCDebug(qLcCodecStorage) << "Skip experimental codec" << codec.name();

++                 continue;

++             }

++

++             if (codec.isDecoder()) {

++                 if (isCodecValid(codec, HWAccel::decodingDeviceTypes(), platformHwDecoders))

++                     result[qToUnderlying(CodecRole::Decoders)].emplace_back(codec);

++                 else

++                     qCDebug(qLcCodecStorage) << "Skip decoder" << codec.name()

++                                              << "due to disabled matching hw acceleration, or "

++                                                 "dysfunctional codec";

++             }

++

++             if (codec.isEncoder()) {

++                 if (isExcludedEncoder(codec.name()))

++                     continue;

++

++                 if (isCodecValid(codec, HWAccel::encodingDeviceTypes(), platformHwEncoders))

++                     result[qToUnderlying(CodecRole::Encoders)].emplace_back(codec);

++                 else

++                     qCDebug(qLcCodecStorage) << "Skip encoder" << codec.name()

++                                              << "due to disabled matching hw acceleration, or "

++                                                 "dysfunctional codec";

++             }

++         }

++

++         for (auto &storage : result) {

++             storage.shrink_to_fit();

++

++             // we should ensure the original order

++             ranges::stable_sort(storage, CodecsComparator{});

++         }

++

++         // It print pretty much logs, so let's print it only for special case

++         const bool shouldDumpCodecsInfo = qLcCodecStorage().isEnabled(QtDebugMsg)

++                 && qEnvironmentVariableIsSet("QT_FFMPEG_DEBUG");

++

++         if (shouldDumpCodecsInfo) {

++             qCDebug(qLcCodecStorage) << "Advanced FFmpeg codecs info:";

++             for (auto &storage : result) {

++                 for (auto &codec : storage)

++                     dumpCodecInfo(codec);

++                 qCDebug(qLcCodecStorage) << "---------------------------";

++             }

++         }

++         return result;

++     }

++

++     CodecStoreSingleton()

++     {

++         // enumerate codecs asynchronously, so that enumeration is done on a separate thread

++         // without COM initialization, as otherwise avcodec_open2 will fail and ffmpeg will

++         // warn that "COM must not be in STA mode"

++         constexpr auto launchPolicy =

++                 QOperatingSystemVersion::currentType() == QOperatingSystemVersion::Windows

++                 ? std::launch::async

++                 : std::launch::deferred;

++

++         codecStoreFuture = std::async(launchPolicy, [] {

++             return enumerateCodecs();

++         }).share();

++     }

++ };

++

++ Q_APPLICATION_STATIC(CodecStoreSingleton, codecStoreSingleton)

++

++ QSpan<const Codec> codecsStorage(CodecRole codecsType)

++ {

++     return codecStoreSingleton->codecStoreFuture.get()[qToUnderlying(codecsType)];

++ }

++

++ std::optional<Codec> findAVCodec(CodecRole codecsType, AVCodecID codecId,

++                                  const std::optional<PixelOrSampleFormat> &format)

++ {

++     namespace ranges = QtMultimediaPrivate::ranges;

++

++     QSpan codecs = codecsStorage(codecsType);

++

++     // Codecs are sorted, so we can quickly narrow down the search to codecs with the specific id.

++     auto codecsForId = ranges::equal_range(codecs, codecId, CodecsComparator{});

++

++     // Within the narrowed down range, look for a codec that supports the format.

++     // If no format is specified, return the first one.

++     auto codecIt = ranges::find_if(codecsForId, [&](const Codec &codec) {

++         return !format || isAVFormatSupported(codec, *format);

++     });

++

++     if (codecIt != codecsForId.end())

++         return *codecIt;

++

++     return {};

++ }

++

++ } // namespace

++

++ std::vector<CodecScoreRecord>

++ findAndScoreCodecs(CodecRole codecsType, AVCodecID codecId,

++                    const qxp::function_ref<AVScore(const Codec &)> &scoreFunction)

++ {

++     namespace ranges = QtMultimediaPrivate::ranges;

++     namespace views = QtMultimediaPrivate::views;

++

++     QSpan codecs = codecsStorage(codecsType);

++     auto codecsForId = ranges::equal_range(codecs, codecId, CodecsComparator{});

++     auto scoredCodecs = views::transform(codecsForId, [&](const Codec &codec) {

++         return CodecScoreRecord{

++             codec,

++             scoreFunction(codec),

++         };

++     });

++     auto validCodecs = views::filter(scoredCodecs, [](const CodecScoreRecord &record) {

++         return record.score != NotSuitableAVScore;

++     });

++

++     auto result = ranges::to<std::vector<CodecScoreRecord>>(validCodecs);

++     ranges::stable_sort(result, [](const CodecScoreRecord &a, const CodecScoreRecord &b) {

++         return a.score > b.score;

++     });

++

++     if (qLcCodecStorage().isEnabled(QtDebugMsg))

++         for (const auto &[codec, score] : result)

++             qCDebug(qLcCodecStorage)

++                     << "findAndOpenCodec(): candidate:" << codec.name() << "score:" << score;

++     return result;

++ }

++

++ std::optional<Codec> findAVDecoder(AVCodecID codecId,

++                                    const std::optional<PixelOrSampleFormat> &format)

++ {

++     return findAVCodec(CodecRole::Decoders, codecId, format);

++ }

++

++ std::optional<Codec> findAVEncoder(AVCodecID codecId, const std::optional<PixelOrSampleFormat> &format)

++ {

++     return findAVCodec(CodecRole::Encoders, codecId, format);

++ }

++

++ } // namespace QFFmpeg

++

++ QT_END_NAMESPACE

Revision control