Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "playbackengine/qffmpegmediadataholder_p.h"
#include "qffmpegmediametadata_p.h"
#include "qffmpegmediaformatinfo_p.h"
#include "qffmpegioutils_p.h"
#include <QtMultimedia/qplaybackoptions.h>
#include <QtMultimedia/private/qmediametadata_p.h>
#include <QtCore/qiodevice.h>
#include <QtCore/qdatetime.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/private/qminimalflatset_p.h>
#if defined(Q_OS_ANDROID)
# include <QtFFmpegMediaPluginImpl/private/qandroidcacertificates_p.h>
#endif
#include <optional>
QT_BEGIN_NAMESPACE
Q_STATIC_LOGGING_CATEGORY(qLcMediaDataHolder, "qt.multimedia.ffmpeg.mediadataholder")
namespace QFFmpeg {
using TrackType = QPlatformMediaPlayer::TrackType;
static std::optional<TrackDuration> streamDuration(const AVStream &stream)
{
if (stream.duration > 0)
return toTrackDuration(AVStreamDuration(stream.duration), &stream);
// In some cases ffmpeg reports negative duration that is definitely invalid.
// However, the correct duration may be read from the metadata.
if (stream.duration < 0 && stream.duration != AV_NOPTS_VALUE) {
qCWarning(qLcMediaDataHolder) << "AVStream duration" << stream.duration
<< "is invalid. Taking it from the metadata";
}
if (const auto duration = av_dict_get(stream.metadata, "DURATION", nullptr, 0)) {
const auto time = QTime::fromString(QString::fromUtf8(duration->value));
return TrackDuration(qint64(1000) * time.msecsSinceStartOfDay());
}
return {};
}
static std::optional<TrackPosition> streamStart(const AVStream &stream,
const AVDemuxerContextUPtr &context)
{
if (stream.start_time != AV_NOPTS_VALUE)
return toTrackPosition(AVStreamPosition(stream.start_time), &stream, context.get());
else
return {};
}
static QTransform displayMatrixToTransform(const int32_t *displayMatrix)
{
// displayMatrix is stored as
//
// . -- X axis
// |
// | | a b u |
// Y | c d v |
// axis | x y w |
//
// where a, b, c, d, x, y are 16.16 fixed-point values,
// and u, v, w are 30.2 point values.
// Only a, b, c, d impacts on mirroring and rotation,
// so it's enough to propagate them to QTransform.
//
// If we were interested in getting proper XY scales,
// we would divide a,b,c,d by 2^16. The whole scale doesn't
// impact mirroring and rotation, so we don't do so.
auto toRotateMirrorValue = [displayMatrix](int index) {
// toRotateScaleValue would be:
// return displayMatrix[index] / qreal(1 << 16);
return displayMatrix[index];
};
return QTransform(toRotateMirrorValue(0), toRotateMirrorValue(1),
toRotateMirrorValue(3), toRotateMirrorValue(4),
0, 0);
}
static VideoTransformation streamTransformation(const AVStream *stream)
{
Q_ASSERT(stream);
using SideDataSize = decltype(AVPacketSideData::size);
constexpr SideDataSize displayMatrixSize = sizeof(int32_t) * 9;
const AVPacketSideData *sideData = streamSideData(stream, AV_PKT_DATA_DISPLAYMATRIX);
if (!sideData || sideData->size < displayMatrixSize)
return {};
const auto displayMatrix = reinterpret_cast<const int32_t *>(sideData->data);
const QTransform transform = displayMatrixToTransform(displayMatrix);
const VideoTransformationOpt result = qVideoTransformationFromMatrix(transform);
if (!result) {
qCWarning(qLcMediaDataHolder)
<< "Video stream contains malformed display matrix" << transform;
return {};
}
return *result;
}
static bool colorTransferSupportsHdr(const AVStream *stream)
{
if (!stream)
return false;
const AVCodecParameters *codecPar = stream->codecpar;
if (!codecPar)
return false;
const QVideoFrameFormat::ColorTransfer colorTransfer = fromAvColorTransfer(codecPar->color_trc);
// Assume that content is using HDR if the color transfer supports high
// dynamic range. The video may still not utilize the extended range,
// but we can't determine the actual range without decoding frames.
return colorTransfer == QVideoFrameFormat::ColorTransfer_ST2084
|| colorTransfer == QVideoFrameFormat::ColorTransfer_STD_B67;
}
VideoTransformation MediaDataHolder::transformation() const
{
// TODO: Add QMediaMetaData::Mirrored and take from it and QMediaMetaData::Orientation:
// int orientation = m_metaData.value(QMediaMetaData::Orientation).toInt();
// return static_cast<QtVideo::Rotation>(orientation);
const int streamIndex = m_currentAVStreamIndex[TrackType::VideoStream];
if (streamIndex < 0)
return {};
return streamTransformation(m_context->streams[streamIndex]);
}
AVFormatContext *MediaDataHolder::avContext()
{
return m_context.get();
}
int MediaDataHolder::currentStreamIndex(TrackType trackType) const
{
return m_currentAVStreamIndex[trackType];
}
static void insertMediaData(QMediaMetaData &metaData, TrackType trackType, const AVStream *stream)
{
Q_ASSERT(stream);
const auto *codecPar = stream->codecpar;
switch (trackType) {
case TrackType::VideoStream:
metaData.insert(QMediaMetaData::VideoBitRate, (int)codecPar->bit_rate);
metaData.insert(QMediaMetaData::VideoCodec,
QVariant::fromValue(QFFmpegMediaFormatInfo::videoCodecForAVCodecId(
codecPar->codec_id)));
metaData.insert(QMediaMetaData::Resolution, QSize(codecPar->width, codecPar->height));
metaData.insert(QMediaMetaData::VideoFrameRate,
qreal(stream->avg_frame_rate.num) / qreal(stream->avg_frame_rate.den));
metaData.insert(QMediaMetaData::Orientation,
QVariant::fromValue(streamTransformation(stream).rotation));
metaData.insert(QMediaMetaData::HasHdrContent, colorTransferSupportsHdr(stream));
break;
case TrackType::AudioStream:
metaData.insert(QMediaMetaData::AudioBitRate, (int)codecPar->bit_rate);
metaData.insert(QMediaMetaData::AudioCodec,
QVariant::fromValue(QFFmpegMediaFormatInfo::audioCodecForAVCodecId(
codecPar->codec_id)));
break;
default:
break;
}
};
std::optional<TrackType> MediaDataHolder::trackTypeFromMediaType(int mediaType)
{
switch (mediaType) {
case AVMEDIA_TYPE_AUDIO:
return TrackType::AudioStream;
case AVMEDIA_TYPE_VIDEO:
return TrackType::VideoStream;
case AVMEDIA_TYPE_SUBTITLE:
return TrackType::SubtitleStream;
default:
return std::nullopt;
}
}
namespace {
q23::expected<AVDemuxerContextUPtr, MediaDataHolder::ContextError>
loadMedia(const QUrl &mediaUrl, QIODevice *stream, const QPlaybackOptions &playbackOptions,
const std::shared_ptr<ICancelToken> &cancelToken)
{
using std::chrono::duration_cast;
using std::chrono::microseconds;
using std::chrono::milliseconds;
const QByteArray url = mediaUrl.toString(QUrl::PreferLocalFile).toUtf8();
AVDemuxerContextUPtr context{ avformat_alloc_context() };
if (stream) {
if (!stream->isOpen()) {
if (!stream->open(QIODevice::ReadOnly))
return q23::unexpected{
MediaDataHolder::ContextError{
QMediaPlayer::ResourceError,
QLatin1String("Could not open source device."),
},
};
}
auto seek = &seekQIODevice;
if (!stream->isSequential()) {
stream->seek(0);
} else {
context->ctx_flags |= AVFMTCTX_UNSEEKABLE;
seek = nullptr;
}
constexpr int bufferSize = 32768;
unsigned char *buffer = (unsigned char *)av_malloc(bufferSize);
context->pb = avio_alloc_context(buffer, bufferSize, false, stream, &readQIODevice, nullptr,
seek);
}
AVDictionaryHolder dict;
using RtmpProtocols =
QMinimalVarLengthFlatSet<std::basic_string_view<char16_t>, 6, std::less<>>;
static const RtmpProtocols rtmpProtocols{
u"rtmp", u"rtmpe", u"rtmps", u"rtmpt", u"rtmpse", u"rtmpte",
};
// for rtmp streams, the `timout` parameter implies acting as a server:
// This is not the semantics of QPlaybackOptions::networkTimeout, and will cause failures when
// opening streams
const bool setNetworkTimeout = !rtmpProtocols.contains(mediaUrl.scheme());
if (setNetworkTimeout) {
const milliseconds timeout = playbackOptions.networkTimeout();
av_dict_set_int(dict, "timeout", duration_cast<microseconds>(timeout).count(), 0);
qCDebug(qLcMediaDataHolder) << "Using custom network timeout:" << timeout;
}
{
const int probeSize = playbackOptions.probeSize();
if (probeSize != -1) {
constexpr int minProbeSizeFFmpeg = 32;
if (probeSize >= minProbeSizeFFmpeg) {
av_dict_set_int(dict, "probesize", probeSize, 0);
qCDebug(qLcMediaDataHolder) << "Using custom probesize" << probeSize;
} else
qCWarning(qLcMediaDataHolder) << "Invalid probe size, using default";
}
}
const QByteArray protocolWhitelist = qgetenv("QT_FFMPEG_PROTOCOL_WHITELIST");
if (!protocolWhitelist.isNull())
av_dict_set(dict, "protocol_whitelist", protocolWhitelist.data(), 0);
if (mediaUrl.scheme().compare(u"rtsp") == 0) {
const QByteArray rtspTransport = qgetenv("QT_FFMPEG_RTSP_TRANSPORT").trimmed();
if (!rtspTransport.isEmpty()) {
av_dict_set(dict, "rtsp_transport", rtspTransport.constData(), 0);
qCDebug(qLcMediaDataHolder) << "Using custom RTSP transport:" << rtspTransport;
}
}
if (playbackOptions.playbackIntent() == QPlaybackOptions::PlaybackIntent::LowLatencyStreaming) {
av_dict_set(dict, "fflags", "nobuffer", 0);
av_dict_set_int(dict, "flush_packets", 1, 0);
qCDebug(qLcMediaDataHolder) << "Enabled low latency streaming";
}
// QTBUG-145590: for hls streams, we want to disable http persistent connections to allow FFmpeg
// (before FFmpeg 8?) to mix raw and encrypted streams
if (avformat_version() < AV_VERSION_INT(62, 12, 100))
av_dict_set_int(dict, "http_persistent", 0, 0);
#ifdef Q_OS_ANDROID
// QTBUG-150720: FFmpeg >= 9.0 defaults its tls protocol to peer certificate verification,
// but we do not supply a CA bundle on Android by default, so every https:// source would
// otherwise fail to open.
// TODO: Apply the app's Network Security Configuration to media sources: domain-specific
// trust anchors, Certificate Transparency and cleartextTrafficPermitted="false". The CA
// bundle only carries the base-config trust anchors, so this requires verifying the peer
// chain per host through the same trust manager QtSslCertificates reads the anchors from
// (via X509TrustManagerExtensions.checkServerTrusted(chain, authType, host)), and rejecting
// cleartext protocols based on NetworkSecurityPolicy.isCleartextTrafficPermitted(host).
{
const QString caBundlePath = androidFFmpegCaBundlePath();
if (!caBundlePath.isEmpty())
av_dict_set(dict, "ca_file", caBundlePath.toUtf8().constData(), 0);
}
#endif
// NOTE: For other platforms, CA bundles in default directories are OK,
// and one can set non-standard locations with SSL_CERT_DIR=<directory>
context->interrupt_callback.opaque = cancelToken.get();
context->interrupt_callback.callback = [](void *opaque) {
const auto *cancelToken = static_cast<const ICancelToken *>(opaque);
if (cancelToken && cancelToken->isCancelled())
return 1;
return 0;
};
int ret = 0;
AVIOContext *customIo = context->pb;
{
AVFormatContext *contextRaw = context.release();
ret = avformat_open_input(&contextRaw, url.constData(), nullptr, dict);
context.reset(contextRaw);
}
if (ret < 0) {
auto code = QMediaPlayer::ResourceError;
if (ret == AVERROR(EACCES))
code = QMediaPlayer::AccessDeniedError;
else if (ret == AVERROR(EINVAL) || ret == AVERROR_INVALIDDATA)
code = QMediaPlayer::FormatError;
qCWarning(qLcMediaDataHolder)
<< "Could not open media. FFmpeg error description:" << AVError(ret);
freeAVIOContext(customIo);
return q23::unexpected{
MediaDataHolder::ContextError{ code, QMediaPlayer::tr("Could not open file") },
};
}
ret = avformat_find_stream_info(context.get(), nullptr);
if (ret < 0) {
return q23::unexpected{
MediaDataHolder::ContextError{
QMediaPlayer::FormatError,
QMediaPlayer::tr("Could not find stream information for media file") },
};
}
if (qLcMediaDataHolder().isInfoEnabled())
av_dump_format(context.get(), 0, url.constData(), 0);
return context;
}
} // namespace
MediaDataHolder::Maybe MediaDataHolder::create(const QUrl &url, QIODevice *stream,
const QPlaybackOptions &options,
const std::shared_ptr<ICancelToken> &cancelToken)
{
q23::expected context = loadMedia(url, stream, options, cancelToken);
if (context) {
// MediaDataHolder is wrapped in a shared pointer to interop with signal/slot mechanism
return std::make_shared<MediaDataHolder>(
MediaDataHolder{ std::move(context.value()), cancelToken });
}
return q23::unexpected{ context.error() };
}
MediaDataHolder::MediaDataHolder(AVDemuxerContextUPtr context,
const std::shared_ptr<ICancelToken> &cancelToken)
: m_cancelToken{ cancelToken }
{
Q_ASSERT(context);
m_context = std::move(context);
m_isSeekable = !(m_context->ctx_flags & AVFMTCTX_UNSEEKABLE);
std::optional<TrackDuration> mediaDuration;
for (unsigned int i = 0; i < m_context->nb_streams; ++i) {
const auto *stream = m_context->streams[i];
const auto trackType = trackTypeFromMediaType(stream->codecpar->codec_type);
if (!trackType)
continue;
if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC)
continue; // Ignore attached picture streams because we treat them as metadata
if (stream->time_base.num <= 0 || stream->time_base.den <= 0) {
// An invalid stream timebase is not expected to be given by FFmpeg
qCWarning(qLcMediaDataHolder) << "A stream for the track type" << trackType
<< "has an invalid timebase:" << stream->time_base;
continue;
}
auto metaData = QFFmpegMetaData::fromAVMetaData(stream->metadata);
const bool isDefault = stream->disposition & AV_DISPOSITION_DEFAULT;
if (trackType != TrackType::SubtitleStream) {
insertMediaData(metaData, *trackType, stream);
auto &requestedStream = m_requestedStreams[*trackType];
if (isDefault && requestedStream < 0)
requestedStream = m_streamMap[*trackType].size();
}
if (auto duration = streamDuration(*stream))
metaData.insert(QMediaMetaData::Duration, toUserDuration(*duration).get());
m_streamMap[*trackType].append({ (int)i, isDefault, metaData });
}
// With some media files, streams may be lacking duration info. Let's
// get it from ffmpeg's duration estimation instead.
if (m_context->duration > 0ll)
mediaDuration = toTrackDuration(AVContextDuration(m_context->duration));
if (!mediaDuration) {
std::optional<AVContextDuration> contextStart = contextStartOffset(m_context.get());
std::optional<TrackPosition> startPosition = [&]() -> std::optional<TrackPosition> {
if (contextStart)
return toTrackDuration(*contextStart).asTimePoint();
return std::nullopt;
}();
std::optional<TrackPosition> endPosition;
QSpan streams{ m_context->streams, qsizetype(m_context->nb_streams) };
for (const AVStream *stream : streams) {
std::optional<TrackPosition> currentStreamStartPosition =
streamStart(*stream, m_context);
std::optional<TrackDuration> currentStreamDuration = streamDuration(*stream);
std::optional<TrackPosition> currentStreamEndPosition =
[&]() -> std::optional<TrackPosition> {
if (currentStreamStartPosition && currentStreamDuration)
return *currentStreamStartPosition + *currentStreamDuration;
return std::nullopt;
}();
if (startPosition)
startPosition = std::min(*startPosition,
currentStreamStartPosition.value_or(*startPosition));
else
startPosition = currentStreamStartPosition;
endPosition = std::max(endPosition, currentStreamEndPosition);
}
if (endPosition && startPosition)
mediaDuration = *endPosition - *startPosition;
}
m_duration = mediaDuration.value_or(TrackDuration::zero());
for (auto trackType : { TrackType::VideoStream,
TrackType::AudioStream }) {
auto &requestedStream = m_requestedStreams[trackType];
auto &streamMap = m_streamMap[trackType];
if (requestedStream < 0 && !streamMap.empty())
requestedStream = 0;
if (requestedStream >= 0)
m_currentAVStreamIndex[trackType] =
streamMap[requestedStream].avStreamIndex;
}
updateMetaData();
}
namespace {
/*!
\internal
Attempt to find an attached picture from the context's streams.
This will find ID3v2 pictures on audio files, and also pictures
attached to videos.
*/
QImage getAttachedPicture(const AVFormatContext *context)
{
if (!context)
return {};
for (unsigned int i = 0; i < context->nb_streams; ++i) {
const AVStream* stream = context->streams[i];
if (!stream || !(stream->disposition & AV_DISPOSITION_ATTACHED_PIC))
continue;
const AVPacket *compressedImage = &stream->attached_pic;
if (!compressedImage || !compressedImage->data || compressedImage->size <= 0)
continue;
// Feed raw compressed data to QImage::fromData, which will decompress it
// if it is a recognized format.
QImage image = QImage::fromData({ compressedImage->data, compressedImage->size });
if (!image.isNull())
return image;
}
return {};
}
} // namespace
void MediaDataHolder::updateMetaData()
{
m_metaData = {};
if (!m_context)
return;
m_metaData = QFFmpegMetaData::fromAVMetaData(m_context->metadata);
m_metaData.insert(QMediaMetaData::FileFormat,
QVariant::fromValue(QFFmpegMediaFormatInfo::fileFormatForAVInputFormat(
*m_context->iformat)));
m_metaData.insert(QMediaMetaData::Duration, toUserDuration(m_duration).get());
if (!m_cachedThumbnail.has_value())
m_cachedThumbnail = getAttachedPicture(m_context.get());
QtMultimediaPrivate::setCoverArtImage(m_metaData, *m_cachedThumbnail);
for (auto trackType : { TrackType::AudioStream,
TrackType::VideoStream }) {
const auto streamIndex = m_currentAVStreamIndex[trackType];
if (streamIndex >= 0)
insertMediaData(m_metaData, trackType, m_context->streams[streamIndex]);
}
}
bool MediaDataHolder::setActiveTrack(TrackType type, int streamNumber)
{
if (!m_context)
return false;
if (streamNumber < 0 || streamNumber >= m_streamMap[type].size())
streamNumber = -1;
if (m_requestedStreams[type] == streamNumber)
return false;
m_requestedStreams[type] = streamNumber;
const int avStreamIndex = m_streamMap[type].value(streamNumber).avStreamIndex;
const int oldIndex = m_currentAVStreamIndex[type];
qCDebug(qLcMediaDataHolder) << ">>>>> change track" << type << "from" << oldIndex << "to"
<< avStreamIndex;
// TODO: maybe add additional verifications
m_currentAVStreamIndex[type] = avStreamIndex;
updateMetaData();
return true;
}
int MediaDataHolder::activeTrack(TrackType type) const
{
return m_requestedStreams[type];
}
const QList<MediaDataHolder::StreamInfo> &MediaDataHolder::streamInfo(
TrackType trackType) const
{
return m_streamMap[trackType];
}
} // namespace QFFmpeg
QT_END_NAMESPACE