Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qgstreamerformatinfo_p.h"
#include <common/qglist_helper_p.h>
#include <QtCore/qset.h>
#include <gst/gst.h>
QT_BEGIN_NAMESPACE
QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructureView structure)
{
using namespace std::string_view_literals;
const char *name = structure.name().data();
if (!name || (strncmp(name, "audio/", 6) != 0))
return QMediaFormat::AudioCodec::Unspecified;
name += 6;
if (name == "mpeg"sv) {
auto version = structure["mpegversion"].toInt();
if (version == 1) {
auto layer = structure["layer"];
if (!layer.isNull())
return QMediaFormat::AudioCodec::MP3;
}
if (version == 4)
return QMediaFormat::AudioCodec::AAC;
return QMediaFormat::AudioCodec::Unspecified;
}
if (name == "x-ac3"sv)
return QMediaFormat::AudioCodec::AC3;
if (name == "x-eac3"sv)
return QMediaFormat::AudioCodec::EAC3;
if (name == "x-flac"sv)
return QMediaFormat::AudioCodec::FLAC;
if (name == "x-alac"sv)
return QMediaFormat::AudioCodec::ALAC;
if (name == "x-true-hd"sv)
return QMediaFormat::AudioCodec::DolbyTrueHD;
if (name == "x-vorbis"sv)
return QMediaFormat::AudioCodec::Vorbis;
if (name == "x-opus"sv)
return QMediaFormat::AudioCodec::Opus;
if (name == "x-wav"sv)
return QMediaFormat::AudioCodec::Wave;
if (name == "x-wma"sv)
return QMediaFormat::AudioCodec::WMA;
return QMediaFormat::AudioCodec::Unspecified;
}
QMediaFormat::VideoCodec QGstreamerFormatInfo::videoCodecForCaps(QGstStructureView structure)
{
using namespace std::string_view_literals;
const char *name = structure.name().data();
if (!name || (strncmp(name, "video/", 6) != 0))
return QMediaFormat::VideoCodec::Unspecified;
name += 6;
if (name == "mpeg"sv) {
auto version = structure["mpegversion"].toInt();
if (version == 1)
return QMediaFormat::VideoCodec::MPEG1;
if (version == 2)
return QMediaFormat::VideoCodec::MPEG2;
if (version == 4)
return QMediaFormat::VideoCodec::MPEG4;
return QMediaFormat::VideoCodec::Unspecified;
}
if (name == "x-h264"sv)
return QMediaFormat::VideoCodec::H264;
if (name == "x-h265"sv)
return QMediaFormat::VideoCodec::H265;
if (name == "x-vp8"sv)
return QMediaFormat::VideoCodec::VP8;
if (name == "x-vp9"sv)
return QMediaFormat::VideoCodec::VP9;
if (name == "x-av1"sv)
return QMediaFormat::VideoCodec::AV1;
if (name == "x-theora"sv)
return QMediaFormat::VideoCodec::Theora;
if (name == "x-jpeg"sv)
return QMediaFormat::VideoCodec::MotionJPEG;
if (name == "x-wmv"sv)
return QMediaFormat::VideoCodec::WMV;
return QMediaFormat::VideoCodec::Unspecified;
}
QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructureView structure)
{
using namespace std::string_view_literals;
const char *name = structure.name().data();
if (name == "video/x-ms-asf"sv)
return QMediaFormat::FileFormat::WMV;
if (name == "video/x-msvideo"sv)
return QMediaFormat::FileFormat::AVI;
if (name == "video/x-matroska"sv)
return QMediaFormat::FileFormat::Matroska;
if (name == "video/quicktime"sv) {
const char *variant = structure["variant"].toString();
if (!variant)
return QMediaFormat::FileFormat::QuickTime;
if (variant == "iso"sv)
return QMediaFormat::FileFormat::MPEG4;
}
if (name == "video/ogg"sv)
return QMediaFormat::FileFormat::Ogg;
if (name == "video/webm"sv)
return QMediaFormat::FileFormat::WebM;
if (name == "audio/x-m4a"sv)
return QMediaFormat::FileFormat::Mpeg4Audio;
if (name == "audio/x-wav"sv)
return QMediaFormat::FileFormat::Wave;
if (name == "audio/mpeg"sv) {
auto mpegversion = structure["mpegversion"].toInt();
if (mpegversion == 1) {
auto layer = structure["layer"];
if (!layer.isNull())
return QMediaFormat::FileFormat::MP3;
}
}
if (name == "audio/aac"sv)
return QMediaFormat::FileFormat::AAC;
if (name == "audio/x-ms-wma"sv)
return QMediaFormat::FileFormat::WMA;
if (name == "audio/x-flac"sv)
return QMediaFormat::FileFormat::FLAC;
return QMediaFormat::UnspecifiedFormat;
}
QImageCapture::FileFormat QGstreamerFormatInfo::imageFormatForCaps(QGstStructureView structure)
{
using namespace std::string_view_literals;
const char *name = structure.name().data();
if (name == "image/jpeg"sv)
return QImageCapture::JPEG;
if (name == "image/png"sv)
return QImageCapture::PNG;
if (name == "image/webp"sv)
return QImageCapture::WebP;
if (name == "image/tiff"sv)
return QImageCapture::Tiff;
return QImageCapture::UnspecifiedFormat;
}
static std::pair<QList<QMediaFormat::AudioCodec>, QList<QMediaFormat::VideoCodec>>
getCodecsList(bool decode)
{
QList<QMediaFormat::AudioCodec> audio;
QList<QMediaFormat::VideoCodec> video;
GstPadDirection padDirection = decode ? GST_PAD_SINK : GST_PAD_SRC;
GList *elementList = gst_element_factory_list_get_elements(decode ? GST_ELEMENT_FACTORY_TYPE_DECODER : GST_ELEMENT_FACTORY_TYPE_ENCODER,
GST_RANK_MARGINAL);
for (GstElementFactory *factory :
QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) {
for (GstStaticPadTemplate *padTemplate :
QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>(
gst_element_factory_get_static_pad_templates(factory))) {
if (padTemplate->direction == padDirection) {
auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef);
for (int i = 0; i < caps.size(); i++) {
QGstStructureView structure = caps.at(i);
auto a = QGstreamerFormatInfo::audioCodecForCaps(structure);
if (a != QMediaFormat::AudioCodec::Unspecified && !audio.contains(a))
audio.append(a);
auto v = QGstreamerFormatInfo::videoCodecForCaps(structure);
if (v != QMediaFormat::VideoCodec::Unspecified && !video.contains(v))
video.append(v);
}
}
}
}
gst_plugin_feature_list_free(elementList);
return {audio, video};
}
QList<QGstreamerFormatInfo::CodecMap>
QGstreamerFormatInfo::getCodecMaps(QMediaFormat::ConversionMode conversionMode,
const QList<QMediaFormat::AudioCodec> &supportedAudioCodecs,
const QList<QMediaFormat::VideoCodec> &supportedVideoCodecs)
{
QList<QGstreamerFormatInfo::CodecMap> maps;
GstPadDirection dataPadDirection =
(conversionMode == QMediaFormat::Decode) ? GST_PAD_SINK : GST_PAD_SRC;
auto encodableFactoryTypes =
(GST_ELEMENT_FACTORY_TYPE_MUXER | GST_ELEMENT_FACTORY_TYPE_PAYLOADER
| GST_ELEMENT_FACTORY_TYPE_ENCRYPTOR | GST_ELEMENT_FACTORY_TYPE_ENCODER);
GList *elementList = gst_element_factory_list_get_elements(
(conversionMode == QMediaFormat::Decode) ? GST_ELEMENT_FACTORY_TYPE_DECODABLE
: encodableFactoryTypes,
GST_RANK_MARGINAL);
for (GstElementFactory *factory :
QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) {
QList<QMediaFormat::FileFormat> fileFormats;
for (GstStaticPadTemplate *padTemplate :
QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>(
gst_element_factory_get_static_pad_templates(factory))) {
// Check pads on data side for file formats, except for parsers check source side
if (padTemplate->direction == dataPadDirection
|| (gst_element_factory_list_is_type(factory, GST_ELEMENT_FACTORY_TYPE_PARSER)
&& padTemplate->direction == GST_PAD_SRC)) {
auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef);
for (int i = 0; i < caps.size(); i++) {
QGstStructureView structure = caps.at(i);
auto fmt = fileFormatForCaps(structure);
if (fmt != QMediaFormat::UnspecifiedFormat)
fileFormats.append(fmt);
}
}
}
if (fileFormats.isEmpty())
continue;
QList<QMediaFormat::AudioCodec> audioCodecs;
QList<QMediaFormat::VideoCodec> videoCodecs;
bool hasAnySrcCaps = false;
for (GstStaticPadTemplate *padTemplate :
QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>(
gst_element_factory_get_static_pad_templates(factory))) {
// check the other side for supported inputs/outputs
if (padTemplate->direction != dataPadDirection) {
auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef);
// Demuxers like matroskademux/qtdemux/asfdemux advertise ANY on their SRC pad
// templates because the actual output caps depend on the stream.
if (gst_caps_is_any(caps.get())) {
hasAnySrcCaps = true;
continue;
}
bool acceptsRawAudio = false;
for (int i = 0; i < caps.size(); i++) {
QGstStructureView structure = caps.at(i);
if (structure.name() == "audio/x-raw")
acceptsRawAudio = true;
auto audio = audioCodecForCaps(structure);
if (audio != QMediaFormat::AudioCodec::Unspecified && supportedAudioCodecs.contains(audio))
audioCodecs.append(audio);
auto video = videoCodecForCaps(structure);
if (video != QMediaFormat::VideoCodec::Unspecified && supportedVideoCodecs.contains(video))
videoCodecs.append(video);
}
if (acceptsRawAudio && fileFormats.size() == 1) {
switch (fileFormats.at(0)) {
case QMediaFormat::Mpeg4Audio:
default:
break;
case QMediaFormat::MP3:
audioCodecs.append(QMediaFormat::AudioCodec::MP3);
break;
case QMediaFormat::FLAC:
audioCodecs.append(QMediaFormat::AudioCodec::FLAC);
break;
case QMediaFormat::Wave:
audioCodecs.append(QMediaFormat::AudioCodec::Wave);
break;
}
}
}
}
// SRC pads reported ANY: the demuxer supports whatever codecs the installed decoders
// handle. We claim to support all discovered audio/video codecs in that case, and rely on fixupCodecMaps() to
// remove unsupported combinations.
if (hasAnySrcCaps) {
if (audioCodecs.isEmpty())
audioCodecs = supportedAudioCodecs;
if (videoCodecs.isEmpty())
videoCodecs = supportedVideoCodecs;
}
if (!audioCodecs.isEmpty() || !videoCodecs.isEmpty()) {
for (auto f : std::as_const(fileFormats)) {
// QTBUG-131726: encodebin rejects muxers for x-ms-asf, so we should ignore WMV/WMA
// for encoding
if ((f == QMediaFormat::WMV || f == QMediaFormat::WMA)
&& conversionMode == QMediaFormat::Encode)
continue;
maps.append({f, audioCodecs, videoCodecs});
if (f == QMediaFormat::MPEG4 && !fileFormats.contains(QMediaFormat::Mpeg4Audio)) {
maps.append({QMediaFormat::Mpeg4Audio, audioCodecs, {}});
if (audioCodecs.contains(QMediaFormat::AudioCodec::AAC))
maps.append({QMediaFormat::AAC, { QMediaFormat::AudioCodec::AAC }, {}});
} else if (f == QMediaFormat::WMV && !fileFormats.contains(QMediaFormat::WMA)) {
maps.append({QMediaFormat::WMA, audioCodecs, {}});
}
}
}
}
gst_plugin_feature_list_free(elementList);
return maps;
}
static QList<QImageCapture::FileFormat> getImageFormatList()
{
QSet<QImageCapture::FileFormat> formats;
GList *elementList = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER,
GST_RANK_MARGINAL);
for (GstElementFactory *factory :
QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) {
for (GstStaticPadTemplate *padTemplate :
QGstUtils::GListConstRangeAdaptor<GstStaticPadTemplate *>(
gst_element_factory_get_static_pad_templates(factory))) {
if (padTemplate->direction == GST_PAD_SRC) {
QGstCaps caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef);
for (int i = 0; i < caps.size(); i++) {
QGstStructureView structure = caps.at(i);
auto f = QGstreamerFormatInfo::imageFormatForCaps(structure);
if (f != QImageCapture::UnspecifiedFormat) {
// qDebug() << structure.toString() << f;
formats.insert(f);
}
}
}
}
}
gst_plugin_feature_list_free(elementList);
return formats.values();
}
#if 0
static void dumpAudioCodecs(const QList<QMediaFormat::AudioCodec> &codecList)
{
qDebug() << "Audio codecs:";
for (const auto &c : codecList)
qDebug() << " " << QMediaFormat::audioCodecName(c);
}
static void dumpVideoCodecs(const QList<QMediaFormat::VideoCodec> &codecList)
{
qDebug() << "Video codecs:";
for (const auto &c : codecList)
qDebug() << " " << QMediaFormat::videoCodecName(c);
}
static void dumpMuxers(const QList<QPlatformMediaFormatInfo::CodecMap> &muxerList)
{
for (const auto &m : muxerList) {
qDebug() << " " << QMediaFormat::fileFormatName(m.format);
qDebug() << " Audio";
for (const auto &a : m.audio)
qDebug() << " " << QMediaFormat::audioCodecName(a);
qDebug() << " Video";
for (const auto &v : m.video)
qDebug() << " " << QMediaFormat::videoCodecName(v);
}
}
#endif
QGstreamerFormatInfo::QGstreamerFormatInfo()
{
auto codecs = getCodecsList(/*decode = */ true);
decoders = getCodecMaps(QMediaFormat::Decode, codecs.first, codecs.second);
codecs = getCodecsList(/*decode = */ false);
encoders = getCodecMaps(QMediaFormat::Encode, codecs.first, codecs.second);
// dumpAudioCodecs(codecs.first);
// dumpVideoCodecs(codecs.second);
// dumpMuxers(encoders);
imageFormats = getImageFormatList();
fixupCodecMaps();
}
QGstreamerFormatInfo::~QGstreamerFormatInfo() = default;
QGstCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const
{
auto format = f.fileFormat();
Q_ASSERT(format != QMediaFormat::UnspecifiedFormat);
const char *capsForFormat[QMediaFormat::LastFileFormat + 1] = {
"video/x-ms-asf", // WMV
"video/x-msvideo", // AVI
"video/x-matroska", // Matroska
"video/quicktime, variant=(string)iso", // MPEG4
"video/ogg", // Ogg
"video/quicktime", // QuickTime
"video/webm", // WebM
"video/quicktime, variant=(string)iso", // Mpeg4Audio is the same is mp4...
"video/quicktime, variant=(string)iso", // AAC is also an MP4 container
"video/x-ms-asf", // WMA, same as WMV
"audio/mpeg, mpegversion=(int)1, layer=(int)3", // MP3
"audio/x-flac", // FLAC
"audio/x-wav" // Wave
};
return QGstCaps(gst_caps_from_string(capsForFormat[format]), QGstCaps::HasRef);
}
QGstCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const
{
auto codec = f.audioCodec();
if (codec == QMediaFormat::AudioCodec::Unspecified)
return {};
const char *capsForCodec[(int)QMediaFormat::AudioCodec::LastAudioCodec + 1] = {
"audio/mpeg, mpegversion=(int)1, layer=(int)3", // MP3
"audio/mpeg, mpegversion=(int)4", // AAC
"audio/x-ac3", // AC3
"audio/x-eac3", // EAC3
"audio/x-flac", // FLAC
"audio/x-true-hd", // DolbyTrueHD
"audio/x-opus", // Opus
"audio/x-vorbis", // Vorbis
"audio/x-raw", // WAVE
"audio/x-wma", // WMA
"audio/x-alac", // ALAC
};
return QGstCaps(gst_caps_from_string(capsForCodec[(int)codec]), QGstCaps::HasRef);
}
QGstCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const
{
auto codec = f.videoCodec();
if (codec == QMediaFormat::VideoCodec::Unspecified)
return {};
const char *capsForCodec[(int)QMediaFormat::VideoCodec::LastVideoCodec + 1] = {
"video/mpeg, mpegversion=(int)1", // MPEG1,
"video/mpeg, mpegversion=(int)2", // MPEG2,
"video/mpeg, mpegversion=(int)4", // MPEG4,
"video/x-h264", // H264,
"video/x-h265", // H265,
"video/x-vp8", // VP8,
"video/x-vp9", // VP9,
"video/x-av1", // AV1,
"video/x-theora", // Theora,
"audio/x-wmv", // WMV
"video/x-jpeg", // MotionJPEG,
};
return QGstCaps(gst_caps_from_string(capsForCodec[(int)codec]), QGstCaps::HasRef);
}
QT_END_NAMESPACE