qt/qtmultimedia/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderoptions.cpp

Revision control

++ // Copyright (C) 2022 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++ #include "qffmpegencoderoptions_p.h"
++
++ #include <QtFFmpegMediaPluginImpl/private/qffmpegmediaformatinfo_p.h>
++ #include <QtFFmpegMediaPluginImpl/private/qffmpegrecordingengineutils_p.h>
++
++ #include <QtMultimedia/qaudioformat.h>
++ #include <QtCore/private/qflatmap_p.h>
++
++ #if QT_CONFIG(vaapi)
++ #include <va/va.h>
++ #endif
++
++ #ifdef Q_OS_ANDROID
++ extern "C" {
++ #include <libavcodec/avcodec.h>
++ }
++ #endif
++
++ #include <libavutil/channel_layout.h>
++
++ QT_BEGIN_NAMESPACE
++
++ // unfortunately there is no common way to specify options for the encoders. The code here tries to map our settings sensibly
++ // to options available in different encoders
++
++ // For constant quality options, we're trying to map things to approx those bit rates for 1080p@30fps (in Mbps):
++ // VeryLow Low Normal High VeryHigh
++ // H264: 0.8M 1.5M 3.5M 6M 10M
++ // H265: 0.5M 1.0M 2.5M 4M 7M
++
++ [[maybe_unused]]
++ static int bitrateForSettings(const QMediaEncoderSettings &settings, bool hdr = false)
++ {
++ // calculate an acceptable bitrate depending on video codec, resolution, framerate and requested quality
++ // The calculations are rather heuristic here, trying to take into account how well codecs compress using
++ // the tables above.
++
++ // The table here is for 30FPS
++ const double bitsPerPixel[int(QMediaFormat::VideoCodec::LastVideoCodec)+1][QMediaRecorder::VeryHighQuality+1] = {
++ { 1.2, 2.25, 5, 9, 15 }, // MPEG1,
++ { 0.8, 1.5, 3.5, 6, 10 }, // MPEG2
++ { 0.4, 0.75, 1.75, 3, 5 }, // MPEG4
++ { 0.4, 0.75, 1.75, 3, 5 }, // H264
++ { 0.3, 0.5, 0.2, 2, 3 }, // H265
++ { 0.4, 0.75, 1.75, 3, 5 }, // VP8
++ { 0.3, 0.5, 0.2, 2, 3 }, // VP9
++ { 0.2, 0.4, 0.9, 1.5, 2.5 }, // AV1
++ { 0.4, 0.75, 1.75, 3, 5 }, // Theora
++ { 0.8, 1.5, 3.5, 6, 10 }, // WMV
++ { 16, 24, 32, 40, 48 }, // MotionJPEG
++ };
++
++ QSize s = settings.videoResolution();
++ double bitrate = bitsPerPixel[int(settings.videoCodec())][settings.quality()]*s.width()*s.height();
++
++ // Frame rate can be 0 with variable-rate sources
++ const double frameRate = settings.videoFrameRate() > 0.
++ ? settings.videoFrameRate()
++ : static_cast<double>(QFFmpeg::DefaultVideoFrameRate);
++
++ if (settings.videoCodec() != QMediaFormat::VideoCodec::MotionJPEG) {
++ // We assume that doubling the framerate requires 1.5 times the amount of data (not twice, as intraframe
++ // differences will be smaller). 4 times the frame rate uses thus 2.25 times the data, etc.
++ float rateMultiplier = log2(frameRate / 30.);
++ bitrate *= pow(1.5, rateMultiplier);
++ } else {
++ // MotionJPEG doesn't optimize between frames, so we have a linear dependency on framerate
++ bitrate *= frameRate / 30.;
++ }
++
++ // HDR requires 10bits per pixel instead of 8, so apply a factor of 1.25.
++ if (hdr)
++ bitrate *= 1.25;
++ return bitrate;
++ }
++
++ static void apply_openh264(const QMediaEncoderSettings &settings, AVCodecContext *codec,
++ AVDictionary **opts)
++ {
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding
++ || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ av_dict_set(opts, "rc_mode", "bitrate", 0);
++ } else {
++ av_dict_set(opts, "rc_mode", "quality", 0);
++ static const int q[] = { 51, 48, 38, 25, 5 };
++ codec->qmax = codec->qmin = q[settings.quality()];
++ }
++ }
++
++ static void apply_x264(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
++ {
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ } else {
++ const char *scales[] = {
++ "29", "26", "23", "21", "19"
++ };
++ av_dict_set(opts, "crf", scales[settings.quality()], 0);
++ }
++ }
++
++ static void apply_x265(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
++ {
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ } else {
++ const char *scales[QMediaRecorder::VeryHighQuality+1] = {
++ "40", "34", "28", "26", "24",
++ };
++ av_dict_set(opts, "crf", scales[settings.quality()], 0);
++ }
++ }
++
++ static void apply_libvpx(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
++ {
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ } else {
++ const char *scales[QMediaRecorder::VeryHighQuality+1] = {
++ "38", "34", "31", "28", "25",
++ };
++ av_dict_set(opts, "crf", scales[settings.quality()], 0);
++ av_dict_set(opts, "b", nullptr, 0);
++ }
++ av_dict_set(opts, "row-mt", "1", 0); // better multithreading
++ }
++
++ static void apply_mpeg4(const QMediaEncoderSettings &settings, AVCodecContext *codec,
++ AVDictionary **opts)
++ {
++
++ QMediaRecorder::EncodingMode encodingMode = settings.encodingMode();
++ switch (encodingMode) {
++ case QMediaRecorder::ConstantBitRateEncoding:
++ case QMediaRecorder::QMediaRecorder::AverageBitRateEncoding: {
++ codec->bit_rate = settings.videoBitRate();
++ if (encodingMode == QMediaRecorder::ConstantBitRateEncoding)
++ codec->rc_min_rate = codec->rc_max_rate = settings.videoBitRate();
++
++ break;
++ }
++ case QMediaRecorder::ConstantQualityEncoding: {
++ constexpr auto scales = std::array{
++ 31, // VeryLowQuality
++ 23, // LowQuality
++ 16, // NormalQuality
++ 9, // HighQuality
++ 1, // VeryHighQuality
++ };
++ av_dict_set_int(opts, "qscale", scales[settings.quality()], 0);
++ codec->global_quality = scales[settings.quality()] * FF_QP2LAMBDA;
++ codec->flags |= AV_CODEC_FLAG_QSCALE;
++ break;
++ }
++ case QMediaRecorder::TwoPassEncoding: {
++ qWarning("Two pass encoding is not supported for MPEG4");
++ break;
++ }
++ default: {
++ Q_UNREACHABLE_RETURN();
++ }
++ }
++ }
++
++ #if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)
++ static void apply_v4l2m2m(const QMediaEncoderSettings &settings, AVCodecContext *codec,
++ AVDictionary ** /*opts*/)
++ {
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding
++ || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ return;
++ }
++
++ // Constant quality: v4l2m2m encoders don't read global_quality/AV_CODEC_FLAG_QSCALE,
++ // so approximate a fixed QP via qmin==qmax over the codec-specific V4L2 QP range.
++ // Bitrate is still required by the driver's frame-level rate controller, so keep
++ // supplying a heuristic ceiling alongside the QP pin.
++ codec->bit_rate = bitrateForSettings(settings);
++
++ const int quality = settings.quality();
++ switch (settings.videoCodec()) {
++ case QMediaFormat::VideoCodec::H264: {
++ static constexpr int q[] = { 45, 38, 28, 20, 10 }; // 0..51
++ codec->qmin = codec->qmax = q[quality];
++ break;
++ }
++ case QMediaFormat::VideoCodec::MPEG4: {
++ static constexpr int q[] = { 28, 22, 16, 10, 4 }; // 1..31
++ codec->qmin = codec->qmax = q[quality];
++ break;
++ }
++ case QMediaFormat::VideoCodec::VP8: {
++ static constexpr int q[] = { 108, 88, 64, 40, 16 }; // 0..127
++ codec->qmin = codec->qmax = q[quality];
++ break;
++ }
++ case QMediaFormat::VideoCodec::VP9: {
++ static constexpr int q[] = { 216, 176, 128, 80, 32 }; // 0..255
++ codec->qmin = codec->qmax = q[quality];
++ break;
++ }
++ default:
++ // e.g. HEVC/H263: driver exposes no QP control, bit_rate heuristic above is all we can do
++ break;
++ }
++ }
++ #endif
++
++ #ifdef Q_OS_DARWIN
++ static void apply_videotoolbox(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
++ {
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ } else {
++ // only use quality on macOS/ARM, as FFmpeg doesn't support it on the other platforms and would throw
++ // an error when initializing the codec
++ #if defined(Q_OS_MACOS) && defined(Q_PROCESSOR_ARM_64)
++ // Videotoolbox describes quality as a number from 0 to 1, with low == 0.25, normal 0.5, high 0.75 and lossless = 1
++ // ffmpeg uses a different scale going from 0 to 11800.
++ // Values here are adjusted to agree approximately with the target bit rates listed above
++ const int scales[] = {
++ 3000, 4800, 5900, 6900, 7700,
++ };
++ codec->global_quality = scales[settings.quality()];
++ codec->flags |= AV_CODEC_FLAG_QSCALE;
++ #else
++ codec->bit_rate = bitrateForSettings(settings);
++ #endif
++ }
++
++ // Videotooldox hw acceleration fails of some hardwares,
++ // allow_sw makes sw encoding available if hw encoding failed.
++ // Under the hood, ffmpeg sets
++ // kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder instead of
++ // kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder
++ av_dict_set(opts, "allow_sw", "1", 0);
++ }
++ #endif
++
++ #if QT_CONFIG(vaapi)
++ static void apply_vaapi(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **/*opts*/)
++ {
++ // See also vaapi_encode_init_rate_control() in libavcodec
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ codec->rc_max_rate = settings.videoBitRate();
++ } else if (settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ } else {
++ const int *quality = nullptr;
++ // unfortunately, all VA codecs use different quality scales :/
++ switch (settings.videoCodec()) {
++ case QMediaFormat::VideoCodec::MPEG2: {
++ static const int q[] = { 20, 15, 10, 8, 6 };
++ quality = q;
++ break;
++ }
++ case QMediaFormat::VideoCodec::MPEG4:
++ case QMediaFormat::VideoCodec::H264: {
++ static const int q[] = { 29, 26, 23, 21, 19 };
++ quality = q;
++ break;
++ }
++ case QMediaFormat::VideoCodec::H265: {
++ static const int q[] = { 40, 34, 28, 26, 24 };
++ quality = q;
++ break;
++ }
++ case QMediaFormat::VideoCodec::VP8: {
++ static const int q[] = { 56, 48, 40, 34, 28 };
++ quality = q;
++ break;
++ }
++ case QMediaFormat::VideoCodec::VP9: {
++ static const int q[] = { 124, 112, 100, 88, 76 };
++ quality = q;
++ break;
++ }
++ case QMediaFormat::VideoCodec::MotionJPEG: {
++ static const int q[] = { 40, 60, 80, 90, 95 };
++ quality = q;
++ break;
++ }
++ case QMediaFormat::VideoCodec::AV1:
++ case QMediaFormat::VideoCodec::Theora:
++ case QMediaFormat::VideoCodec::WMV:
++ default:
++ break;
++ }
++
++ if (quality)
++ codec->global_quality = quality[settings.quality()];
++ }
++ }
++ #endif
++
++ static void apply_nvenc(const QMediaEncoderSettings &settings, AVCodecContext *codec,
++ AVDictionary **opts)
++ {
++ switch (settings.encodingMode()) {
++ case QMediaRecorder::EncodingMode::AverageBitRateEncoding:
++ av_dict_set(opts, "vbr", "1", 0);
++ codec->bit_rate = settings.videoBitRate();
++ break;
++ case QMediaRecorder::EncodingMode::ConstantBitRateEncoding:
++ av_dict_set(opts, "cbr", "1", 0);
++ codec->bit_rate = settings.videoBitRate();
++ codec->rc_max_rate = codec->rc_min_rate = codec->bit_rate;
++ break;
++ case QMediaRecorder::EncodingMode::ConstantQualityEncoding: {
++ static const char *q[] = { "51", "48", "35", "15", "1" };
++ av_dict_set(opts, "cq", q[settings.quality()], 0);
++ } break;
++ default:
++ break;
++ }
++ }
++
++ #ifdef Q_OS_WINDOWS
++ static void apply_mf(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
++ {
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
++ codec->bit_rate = settings.videoBitRate();
++ av_dict_set(opts, "rate_control", "cbr", 0);
++ } else {
++ av_dict_set(opts, "rate_control", "quality", 0);
++ const char *scales[] = {
++ "25", "50", "75", "90", "100"
++ };
++ av_dict_set(opts, "quality", scales[settings.quality()], 0);
++ }
++ }
++ #endif
++
++ #ifdef Q_OS_ANDROID
++ static void apply_mediacodec(const QMediaEncoderSettings &settings, AVCodecContext *codec,
++ AVDictionary **opts)
++ {
++ if (settings.videoBitRate() != -1)
++ codec->bit_rate = settings.videoBitRate();
++ else
++ codec->bit_rate = bitrateForSettings(settings);
++
++ const int quality[] = { 25, 50, 75, 90, 100 };
++ codec->global_quality = quality[settings.quality()];
++
++ switch (settings.encodingMode()) {
++ case QMediaRecorder::EncodingMode::AverageBitRateEncoding:
++ av_dict_set(opts, "bitrate_mode", "vbr", 1);
++ break;
++ case QMediaRecorder::EncodingMode::ConstantBitRateEncoding:
++ av_dict_set(opts, "bitrate_mode", "cbr", 1);
++ break;
++ case QMediaRecorder::EncodingMode::ConstantQualityEncoding:
++ // av_dict_set(opts, "bitrate_mode", "cq", 1);
++ av_dict_set(opts, "bitrate_mode", "cbr", 1);
++ break;
++ default:
++ break;
++ }
++
++ switch (settings.videoCodec()) {
++ case QMediaFormat::VideoCodec::H264: {
++ const char *levels[] = { "2.2", "3.2", "4.2", "5.2", "6.2" };
++ av_dict_set(opts, "level", levels[settings.quality()], 1);
++ codec->profile = AV_PROFILE_H264_HIGH;
++ break;
++ }
++ case QMediaFormat::VideoCodec::H265: {
++ // Set the level only for FFmpeg versions that correctly recognize level values.
++ if (avcodec_version() < 4000612 || avcodec_version() > 4002660) {
++ const char *levels[] = { "h2.1", "h3.1", "h4.1", "h5.1", "h6.1" };
++ av_dict_set(opts, "level", levels[settings.quality()], 1);
++ }
++
++ codec->profile = AV_PROFILE_HEVC_MAIN;
++ break;
++ }
++ default:
++ break;
++ }
++ }
++ #endif
++
++ namespace QFFmpeg {
++
++ using ApplyVideoCodecOptions = void (*)(const QMediaEncoderSettings &settings,
++ AVCodecContext *codec, AVDictionary **opts);
++
++ using namespace Qt::StringLiterals;
++
++ using VideoCodecOptionsTableType =
++ QFlatMap<QLatin1StringView, ApplyVideoCodecOptions, std::less<>,
++ std::vector<QLatin1StringView>, std::vector<ApplyVideoCodecOptions>>;
++
++ const VideoCodecOptionsTableType videoCodecOptionTable{
++ { "libx264"_L1, apply_x264 },
++ { "libx265"_L1, apply_x265 },
++ { "libvpx"_L1, apply_libvpx },
++ { "libvpx_vp9"_L1, apply_libvpx },
++ { "libopenh264"_L1, apply_openh264 },
++ { "h264_nvenc"_L1, apply_nvenc },
++ { "hevc_nvenc"_L1, apply_nvenc },
++ { "av1_nvenc"_L1, apply_nvenc },
++ { "mpeg4"_L1, apply_mpeg4 },
++ #if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)
++ { "h264_v4l2m2m"_L1, apply_v4l2m2m },
++ { "hevc_v4l2m2m"_L1, apply_v4l2m2m },
++ { "mpeg4_v4l2m2m"_L1, apply_v4l2m2m },
++ { "vp8_v4l2m2m"_L1, apply_v4l2m2m },
++ #endif
++ #ifdef Q_OS_DARWIN
++ { "h264_videotoolbox"_L1, apply_videotoolbox },
++ { "hevc_videotoolbox"_L1, apply_videotoolbox },
++ { "prores_videotoolbox"_L1, apply_videotoolbox },
++ { "vp9_videotoolbox"_L1, apply_videotoolbox },
++ #endif
++ #if QT_CONFIG(vaapi)
++ { "mpeg2_vaapi"_L1, apply_vaapi },
++ { "mjpeg_vaapi"_L1, apply_vaapi },
++ { "h264_vaapi"_L1, apply_vaapi },
++ { "hevc_vaapi"_L1, apply_vaapi },
++ { "vp8_vaapi"_L1, apply_vaapi },
++ { "vp9_vaapi"_L1, apply_vaapi },
++ #endif
++ #ifdef Q_OS_WINDOWS
++ { "hevc_mf"_L1, apply_mf },
++ { "h264_mf"_L1, apply_mf },
++ #endif
++ #ifdef Q_OS_ANDROID
++ { "hevc_mediacodec"_L1, apply_mediacodec },
++ { "h264_mediacodec"_L1, apply_mediacodec },
++ #endif
++
++ };
++
++ void applyVideoEncoderOptions(const QMediaEncoderSettings &settings, QLatin1StringView codecName,
++ AVCodecContext *codec, AVDictionary **opts)
++ {
++ av_dict_set(opts, "threads", "auto", 0); // we always want automatic threading
++
++ auto entry = videoCodecOptionTable.find(codecName);
++
++ if (entry != videoCodecOptionTable.end())
++ entry->second(settings, codec, opts);
++ }
++
++ void applyAudioEncoderOptions(const QMediaEncoderSettings &settings,
++ QLatin1StringView /*codecName*/, AVCodecContext *codec,
++ AVDictionary ** /*opts*/)
++ {
++ codec->thread_count = -1; // we always want automatic threading
++ if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding
++ || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding)
++ codec->bit_rate = settings.audioBitRate();
++
++ if (settings.audioSampleRate() != -1)
++ codec->sample_rate = settings.audioSampleRate();
++
++ if (settings.audioChannelCount() != -1) {
++ auto mask = QFFmpegMediaFormatInfo::avChannelLayout(
++ QAudioFormat::defaultChannelConfigForChannelCount(settings.audioChannelCount()));
++
++ #if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
++ av_channel_layout_from_mask(&codec->ch_layout, mask);
++ #else
++ codec->channel_layout = mask;
++ codec->channels = qPopulationCount(codec->channel_layout);
++ #endif
++ }
++ }
++
++ } // namespace QFFmpeg
++
++ QT_END_NAMESPACE