Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qdarwinformatsinfo_p.h"
#include <QtMultimedia/private/qapple_utils_p.h>
#include <QtMultimedia/private/qmultimedia_ranges_p.h>
#include <QtCore/qdebug.h>
#include <QtCore/private/qcore_mac_p.h>
#include <AVFoundation/AVFoundation.h>
#include <AudioToolbox/AudioToolbox.h>
#include <VideoToolbox/VideoToolbox.h>
QT_BEGIN_NAMESPACE
namespace {
constexpr struct
{
const char *__nonnull name;
QMediaFormat::FileFormat fileFormat;
// For single-codec containers the codec is implicit in the MIME type itself — no extended
// MIME type probe needed. Unspecified means probe via audioCodecMap instead.
QMediaFormat::AudioCodec implicitCodec;
} mediaContainerMap[] = {
{ "video/x-ms-asf", QMediaFormat::WMV, QMediaFormat::AudioCodec::Unspecified },
{ "video/avi", QMediaFormat::AVI, QMediaFormat::AudioCodec::Unspecified },
{ "video/x-matroska", QMediaFormat::Matroska, QMediaFormat::AudioCodec::Unspecified },
{ "video/mp4", QMediaFormat::MPEG4, QMediaFormat::AudioCodec::Unspecified },
{ "video/quicktime", QMediaFormat::QuickTime, QMediaFormat::AudioCodec::Unspecified },
{ "audio/ogg", QMediaFormat::Ogg, QMediaFormat::AudioCodec::Unspecified },
{ "audio/mp4", QMediaFormat::Mpeg4Audio, QMediaFormat::AudioCodec::Unspecified },
{ "audio/aac", QMediaFormat::AAC, QMediaFormat::AudioCodec::AAC },
{ "audio/mp3", QMediaFormat::MP3, QMediaFormat::AudioCodec::MP3 },
{ "audio/flac", QMediaFormat::FLAC, QMediaFormat::AudioCodec::FLAC },
{ "audio/x-wav", QMediaFormat::Wave, QMediaFormat::AudioCodec::Wave },
};
constexpr struct {
const char *__nonnull name;
QMediaFormat::VideoCodec codec;
} videoCodecMap[] = {
// See CMVideoCodecType for the four character code names of codecs
{ "; codecs=\"mp1v\"", QMediaFormat::VideoCodec::MPEG1 },
{ "; codecs=\"mp2v\"", QMediaFormat::VideoCodec::MPEG2 },
{ "; codecs=\"mp4v\"", QMediaFormat::VideoCodec::MPEG4 },
{ "; codecs=\"avc1\"", QMediaFormat::VideoCodec::H264 },
{ "; codecs=\"hvc1\"", QMediaFormat::VideoCodec::H265 },
{ "; codecs=\"vp09\"", QMediaFormat::VideoCodec::VP9 },
{ "; codecs=\"av01\"", QMediaFormat::VideoCodec::AV1 },
{ "; codecs=\"jpeg\"", QMediaFormat::VideoCodec::MotionJPEG },
};
constexpr struct {
const char *__nonnull name;
QMediaFormat::AudioCodec codec;
} audioCodecMap[] = {
// RFC 6381 / ISO 14496-3 codec strings understood by isPlayableExtendedMIMEType
{ "; codecs=\"mp4a.40.2\"", QMediaFormat::AudioCodec::AAC }, // AAC-LC
{ "; codecs=\"ac-3\"", QMediaFormat::AudioCodec::AC3 },
{ "; codecs=\"ec-3\"", QMediaFormat::AudioCodec::EAC3 },
{ "; codecs=\"flac\"", QMediaFormat::AudioCodec::FLAC },
{ "; codecs=\"alac\"", QMediaFormat::AudioCodec::ALAC },
{ "; codecs=\"opus\"", QMediaFormat::AudioCodec::Opus },
{ "; codecs=\"vorbis\"", QMediaFormat::AudioCodec::Vorbis },
};
bool canEncodeAudioFormat(AudioFormatID formatID)
{
if (formatID == kAudioFormatLinearPCM)
return true;
UInt32 size = 0;
return AudioFormatGetPropertyInfo(kAudioFormatProperty_Encoders,
sizeof(formatID), &formatID, &size) == noErr
&& size > 0;
}
bool canEncodeDecodeFormat(AudioFormatID formatID)
{
if (formatID == kAudioFormatLinearPCM)
return true;
UInt32 size = 0;
return AudioFormatGetPropertyInfo(kAudioFormatProperty_Decoders, sizeof(formatID), &formatID,
&size)
== noErr
&& size > 0;
}
std::set<CMVideoCodecType> availableVTVideoEncoderCodecTypes()
{
using namespace QtMultimediaPrivate;
QCFType<CFArrayRef> list;
if (VTCopyVideoEncoderList(nullptr, &list) != noErr || !list)
return {};
auto results =
CFArrayRange<CFDictionaryRef>(list) | views::transform([](CFDictionaryRef entry) {
auto *typeRef = CFNumberRef(CFDictionaryGetValue(entry, kVTVideoEncoderList_CodecType));
CMVideoCodecType type = 0;
CFNumberGetValue(typeRef, kCFNumberSInt32Type, &type);
return type;
});
return results | ranges::to<std::set<CMVideoCodecType>>();
}
constexpr bool isVideoContainer(QMediaFormat::FileFormat fileFormat)
{
return fileFormat < QMediaFormat::Mpeg4Audio;
}
bool isPlayableExtendedMIMEType(const char *extendedMimetype)
{
return [AVURLAsset isPlayableExtendedMIMEType:[NSString stringWithUTF8String:extendedMimetype]];
}
bool isPlayableExtendedMIMEType(const char *containerName, const char *codecName)
{
const QByteArray extendedMimetype = QByteArrayView(containerName) + codecName;
return isPlayableExtendedMIMEType(extendedMimetype.constData());
}
constexpr auto recordGetCodec = QtMultimediaPrivate::views::transform([](const auto &record) {
return record.codec;
});
QList<QPlatformMediaFormatInfo::CodecMap> enumerateDecoders()
{
namespace views = QtMultimediaPrivate::views;
namespace ranges = QtMultimediaPrivate::ranges;
using AudioCodec = QMediaFormat::AudioCodec;
using VideoCodec = QMediaFormat::VideoCodec;
using CodecMap = QPlatformMediaFormatInfo::CodecMap;
const bool canDecodeMp3 = canEncodeDecodeFormat(kAudioFormatMPEGLayer3);
QList<CodecMap> decoders;
auto avtypes = [AVURLAsset audiovisualMIMETypes];
for (AVFileType filetype : avtypes) {
for (auto [containerName, fileFormat, implicitCodec] : mediaContainerMap) {
if (strcmp(filetype.UTF8String, containerName))
continue;
QList<VideoCodec> video;
QList<AudioCodec> audio;
if (implicitCodec != QMediaFormat::AudioCodec::Unspecified) {
// Single-codec container: the MIME type appearing in audiovisualMIMETypes is
// sufficient proof of support — no extended MIME type probe needed.
audio << implicitCodec;
} else {
// Multi-codec container: probe each codec via extended MIME type.
if (isVideoContainer(fileFormat)) {
switch (fileFormat) {
case QMediaFormat::AVI: {
// isPlayableExtendedMIMEType returns false for AVI+video combinations even
// when AVFoundation can actually decode them.
if (!isPlayableExtendedMIMEType(containerName))
break;
// AVI is a legacy container format that can carry a wide variety of codecs.
// Hardcode the known supported codecs instead of probing via extended MIME type.
video << VideoCodec::MPEG4 << VideoCodec::H264 << VideoCodec::MotionJPEG;
break;
}
default: {
video = videoCodecMap | views::filter([&](const auto &record) {
return isPlayableExtendedMIMEType(containerName, record.name);
}) | recordGetCodec
| ranges::to<QList<VideoCodec>>();
}
};
}
audio = audioCodecMap | views::filter([&](const auto &record) {
return isPlayableExtendedMIMEType(containerName, record.name);
}) | recordGetCodec
| ranges::to<QList<AudioCodec>>();
// MP3 has no RFC 6381 codec string that isPlayableExtendedMIMEType recognises
// (neither ".mp3" nor "mp4a.6B" work). Add it manually for containers that
// are known to carry it: MPEG-4 family and Mpeg4Audio.
if (canDecodeMp3
&& (isVideoContainer(fileFormat) || fileFormat == QMediaFormat::Mpeg4Audio))
audio << AudioCodec::MP3;
}
decoders << CodecMap{
fileFormat,
std::move(audio),
std::move(video),
};
}
}
return decoders;
}
QList<QPlatformMediaFormatInfo::CodecMap> enumerateEncoders()
{
namespace views = QtMultimediaPrivate::views;
namespace ranges = QtMultimediaPrivate::ranges;
using AudioCodec = QMediaFormat::AudioCodec;
using VideoCodec = QMediaFormat::VideoCodec;
using CodecMap = QPlatformMediaFormatInfo::CodecMap;
static constexpr struct AudioProbeRecord
{
AudioFormatID fmtId;
AudioCodec codec;
} audioProbes[] = {
{ kAudioFormatMPEG4AAC, AudioCodec::AAC },
{ kAudioFormatMPEGLayer3, AudioCodec::MP3 },
{ kAudioFormatAC3, AudioCodec::AC3 },
{ kAudioFormatEnhancedAC3, AudioCodec::EAC3 },
{ kAudioFormatFLAC, AudioCodec::FLAC },
{ kAudioFormatAppleLossless, AudioCodec::ALAC },
{ kAudioFormatOpus, AudioCodec::Opus },
{ kAudioFormatLinearPCM, AudioCodec::Wave },
// no API constants, but the fourcc is supported
{ 'vorb', QMediaFormat::AudioCodec::Vorbis },
};
static constexpr struct VideoProbeRecord
{
CMVideoCodecType type;
VideoCodec codec;
} videoProbes[] = {
{ kCMVideoCodecType_H264, VideoCodec::H264 },
{ kCMVideoCodecType_HEVC, VideoCodec::H265 },
{ kCMVideoCodecType_JPEG, VideoCodec::MotionJPEG },
{ kCMVideoCodecType_MPEG4Video, VideoCodec::MPEG4 },
{ kCMVideoCodecType_AV1, VideoCodec::AV1 },
// no API constants, but the fourcc is supported
{ 'mp1v', VideoCodec::MPEG1 },
{ 'mp2v', VideoCodec::MPEG2 },
};
auto filterCodecs = [](const auto &codecsSet) {
return views::filter([&](auto c) {
return codecsSet.count(c);
});
};
const std::set<AudioCodec> supportedAudioCodecs = audioProbes
| views::filter([](const auto &record) {
return canEncodeAudioFormat(record.fmtId);
}) | recordGetCodec
| ranges::to<std::set<AudioCodec>>();
const std::set<CMVideoCodecType> vtCodecTypes = availableVTVideoEncoderCodecTypes();
const std::set<VideoCodec> supportedVideoCodecs = videoProbes
| views::filter([&](const auto &record) {
return ranges::contains(vtCodecTypes, record.type);
}) | recordGetCodec
| ranges::to<std::set<VideoCodec>>();
auto audioFormats = [&](auto candidates) -> QList<AudioCodec> {
return candidates | filterCodecs(supportedAudioCodecs) | ranges::to<QList<AudioCodec>>();
};
auto videoFormats = [&](auto candidates) -> QList<VideoCodec> {
return candidates | filterCodecs(supportedVideoCodecs) | ranges::to<QList<VideoCodec>>();
};
// Video containers share the same codec candidates; audio candidates differ by container
// spec (e.g. Mpeg4Audio is audio-only, Wave only accepts LPCM).
const QList allSupportedVideoCodecs = videoFormats(videoProbes | recordGetCodec);
const QList allSupportedAudioCodecs = audioFormats(audioProbes | recordGetCodec);
return QList{
CodecMap{ QMediaFormat::MPEG4, allSupportedAudioCodecs, allSupportedVideoCodecs },
CodecMap{ QMediaFormat::QuickTime, allSupportedAudioCodecs, allSupportedVideoCodecs },
CodecMap{ QMediaFormat::Mpeg4Audio,
audioFormats(std::array{ AudioCodec::AAC, AudioCodec::ALAC, AudioCodec::FLAC }),
{} },
CodecMap{ QMediaFormat::Wave, audioFormats(std::array{ AudioCodec::Wave }), {} },
};
}
} // namespace
QDarwinFormatInfo::QDarwinFormatInfo()
{
decoders = enumerateDecoders();
encoders = enumerateEncoders();
// ###
imageFormats << QImageCapture::JPEG;
fixupCodecMaps();
}
QDarwinFormatInfo::~QDarwinFormatInfo() = default;
int QDarwinFormatInfo::audioFormatForCodec(QMediaFormat::AudioCodec codec)
{
int codecId = kAudioFormatMPEG4AAC;
switch (codec) {
case QMediaFormat::AudioCodec::Unspecified:
case QMediaFormat::AudioCodec::DolbyTrueHD:
case QMediaFormat::AudioCodec::Vorbis:
case QMediaFormat::AudioCodec::WMA:
// Unsupported, shouldn't happen. Fall back to AAC
case QMediaFormat::AudioCodec::AAC:
codecId = kAudioFormatMPEG4AAC;
break;
case QMediaFormat::AudioCodec::MP3:
codecId = kAudioFormatMPEGLayer3;
break;
case QMediaFormat::AudioCodec::AC3:
codecId = kAudioFormatAC3;
break;
case QMediaFormat::AudioCodec::EAC3:
codecId = kAudioFormatEnhancedAC3;
break;
case QMediaFormat::AudioCodec::FLAC:
codecId = kAudioFormatFLAC;
break;
case QMediaFormat::AudioCodec::ALAC:
codecId = kAudioFormatAppleLossless;
break;
case QMediaFormat::AudioCodec::Opus:
codecId = kAudioFormatOpus;
break;
case QMediaFormat::AudioCodec::Wave:
codecId = kAudioFormatLinearPCM;
}
return codecId;
}
NSString *QDarwinFormatInfo::videoFormatForCodec(QMediaFormat::VideoCodec codec)
{
switch (codec) {
case QMediaFormat::VideoCodec::MPEG1:
return @"mp1v";
break;
case QMediaFormat::VideoCodec::MPEG2:
return @"mp2v";
break;
case QMediaFormat::VideoCodec::MPEG4:
return @"mp4v";
break;
case QMediaFormat::VideoCodec::H264:
return @"avc1";
break;
case QMediaFormat::VideoCodec::VP9:
return @"vp09";
break;
case QMediaFormat::VideoCodec::MotionJPEG:
return @"jpeg";
case QMediaFormat::VideoCodec::H265:
return @"hvc1";
case QMediaFormat::VideoCodec::AV1:
return @"av01";
case QMediaFormat::VideoCodec::Unspecified:
case QMediaFormat::VideoCodec::VP8:
case QMediaFormat::VideoCodec::Theora:
case QMediaFormat::VideoCodec::WMV:
default:
qDebug() << "Unsupported video codec" << codec << ", using H.265 as fallback";
return @"hvc1";
}
}
NSString *QDarwinFormatInfo::avFileTypeForContainerFormat(QMediaFormat::FileFormat container)
{
switch (container) {
case QMediaFormat::MPEG4:
return AVFileTypeMPEG4;
case QMediaFormat::QuickTime:
return AVFileTypeQuickTimeMovie;
case QMediaFormat::MP3:
return AVFileTypeMPEGLayer3;
case QMediaFormat::Mpeg4Audio:
return AVFileTypeAppleM4A;
case QMediaFormat::Wave:
return AVFileTypeWAVE;
default:
return AVFileTypeQuickTimeMovie;
}
}
QT_END_NAMESPACE