qffmpegmediadataholder.cpp

++ // Copyright (C) 2021 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++

++ #include "playbackengine/qffmpegmediadataholder_p.h"

++

++ #include "qffmpegmediametadata_p.h"

++ #include "qffmpegmediaformatinfo_p.h"

++ #include "qffmpegioutils_p.h"

++

++ #include <QtMultimedia/qplaybackoptions.h>

++ #include <QtMultimedia/private/qmediametadata_p.h>

++ #include <QtCore/qiodevice.h>

++ #include <QtCore/qdatetime.h>

++ #include <QtCore/qloggingcategory.h>

++ #include <QtCore/private/qminimalflatset_p.h>

++

++ #if defined(Q_OS_ANDROID)

++ #  include <QtFFmpegMediaPluginImpl/private/qandroidcacertificates_p.h>

++ #endif

++

++ #include <optional>

++

++ QT_BEGIN_NAMESPACE

++

++ Q_STATIC_LOGGING_CATEGORY(qLcMediaDataHolder, "qt.multimedia.ffmpeg.mediadataholder")

++

++ namespace QFFmpeg {

++

++ using TrackType = QPlatformMediaPlayer::TrackType;

++

++ static std::optional<TrackDuration> streamDuration(const AVStream &stream)

++ {

++     if (stream.duration > 0)

++         return toTrackDuration(AVStreamDuration(stream.duration), &stream);

++

++     // In some cases ffmpeg reports negative duration that is definitely invalid.

++     // However, the correct duration may be read from the metadata.

++

++     if (stream.duration < 0 && stream.duration != AV_NOPTS_VALUE) {

++         qCWarning(qLcMediaDataHolder) << "AVStream duration" << stream.duration

++                                       << "is invalid. Taking it from the metadata";

++     }

++

++     if (const auto duration = av_dict_get(stream.metadata, "DURATION", nullptr, 0)) {

++         const auto time = QTime::fromString(QString::fromUtf8(duration->value));

++         return TrackDuration(qint64(1000) * time.msecsSinceStartOfDay());

++     }

++

++     return {};

++ }

++

++ static std::optional<TrackPosition> streamStart(const AVStream &stream,

++                                                 const AVDemuxerContextUPtr &context)

++ {

++     if (stream.start_time != AV_NOPTS_VALUE)

++         return toTrackPosition(AVStreamPosition(stream.start_time), &stream, context.get());

++     else

++         return {};

++ }

++

++ static QTransform displayMatrixToTransform(const int32_t *displayMatrix)

++ {

++     // displayMatrix is stored as

++     //

++     //  . -- X axis

++     //  |

++     //  |    | a b u |

++     //  Y    | c d v |

++     // axis  | x y w |

++     //

++     // where a, b, c, d, x, y are 16.16 fixed-point values,

++     // and u, v, w are 30.2 point values.

++     // Only a, b, c, d impacts on mirroring and rotation,

++     // so it's enough to propagate them to QTransform.

++     //

++     // If we were interested in getting proper XY scales,

++     // we would divide a,b,c,d by 2^16. The whole scale doesn't

++     // impact mirroring and rotation, so we don't do so.

++

++     auto toRotateMirrorValue = [displayMatrix](int index) {

++         // toRotateScaleValue would be:

++         // return displayMatrix[index] / qreal(1 << 16);

++         return displayMatrix[index];

++     };

++

++     return QTransform(toRotateMirrorValue(0), toRotateMirrorValue(1),

++                       toRotateMirrorValue(3), toRotateMirrorValue(4),

++                       0, 0);

++ }

++

++ static VideoTransformation streamTransformation(const AVStream *stream)

++ {

++     Q_ASSERT(stream);

++

++     using SideDataSize = decltype(AVPacketSideData::size);

++     constexpr SideDataSize displayMatrixSize = sizeof(int32_t) * 9;

++     const AVPacketSideData *sideData = streamSideData(stream, AV_PKT_DATA_DISPLAYMATRIX);

++     if (!sideData || sideData->size < displayMatrixSize)

++         return {};

++

++     const auto displayMatrix = reinterpret_cast<const int32_t *>(sideData->data);

++     const QTransform transform = displayMatrixToTransform(displayMatrix);

++     const VideoTransformationOpt result = qVideoTransformationFromMatrix(transform);

++     if (!result) {

++         qCWarning(qLcMediaDataHolder)

++                 << "Video stream contains malformed display matrix" << transform;

++         return {};

++     }

++     return *result;

++ }

++

++ static bool colorTransferSupportsHdr(const AVStream *stream)

++ {

++     if (!stream)

++         return false;

++

++     const AVCodecParameters *codecPar = stream->codecpar;

++     if (!codecPar)

++         return false;

++

++     const QVideoFrameFormat::ColorTransfer colorTransfer = fromAvColorTransfer(codecPar->color_trc);

++

++     // Assume that content is using HDR if the color transfer supports high

++     // dynamic range. The video may still not utilize the extended range,

++     // but we can't determine the actual range without decoding frames.

++     return colorTransfer == QVideoFrameFormat::ColorTransfer_ST2084

++             || colorTransfer == QVideoFrameFormat::ColorTransfer_STD_B67;

++ }

++

++ VideoTransformation MediaDataHolder::transformation() const

++ {

++     // TODO: Add QMediaMetaData::Mirrored and take from it and QMediaMetaData::Orientation:

++     // int orientation = m_metaData.value(QMediaMetaData::Orientation).toInt();

++     // return static_cast<QtVideo::Rotation>(orientation);

++

++     const int streamIndex = m_currentAVStreamIndex[TrackType::VideoStream];

++     if (streamIndex < 0)

++         return {};

++

++     return streamTransformation(m_context->streams[streamIndex]);

++ }

++

++ AVFormatContext *MediaDataHolder::avContext()

++ {

++     return m_context.get();

++ }

++

++ int MediaDataHolder::currentStreamIndex(TrackType trackType) const

++ {

++     return m_currentAVStreamIndex[trackType];

++ }

++

++ static void insertMediaData(QMediaMetaData &metaData, TrackType trackType, const AVStream *stream)

++ {

++     Q_ASSERT(stream);

++     const auto *codecPar = stream->codecpar;

++

++     switch (trackType) {

++     case TrackType::VideoStream:

++         metaData.insert(QMediaMetaData::VideoBitRate, (int)codecPar->bit_rate);

++         metaData.insert(QMediaMetaData::VideoCodec,

++                         QVariant::fromValue(QFFmpegMediaFormatInfo::videoCodecForAVCodecId(

++                                 codecPar->codec_id)));

++         metaData.insert(QMediaMetaData::Resolution, QSize(codecPar->width, codecPar->height));

++         metaData.insert(QMediaMetaData::VideoFrameRate,

++                         qreal(stream->avg_frame_rate.num) / qreal(stream->avg_frame_rate.den));

++         metaData.insert(QMediaMetaData::Orientation,

++                         QVariant::fromValue(streamTransformation(stream).rotation));

++         metaData.insert(QMediaMetaData::HasHdrContent, colorTransferSupportsHdr(stream));

++         break;

++     case TrackType::AudioStream:

++         metaData.insert(QMediaMetaData::AudioBitRate, (int)codecPar->bit_rate);

++         metaData.insert(QMediaMetaData::AudioCodec,

++                         QVariant::fromValue(QFFmpegMediaFormatInfo::audioCodecForAVCodecId(

++                                 codecPar->codec_id)));

++         break;

++     default:

++         break;

++     }

++ };

++

++ std::optional<TrackType> MediaDataHolder::trackTypeFromMediaType(int mediaType)

++ {

++     switch (mediaType) {

++     case AVMEDIA_TYPE_AUDIO:

++         return TrackType::AudioStream;

++     case AVMEDIA_TYPE_VIDEO:

++         return TrackType::VideoStream;

++     case AVMEDIA_TYPE_SUBTITLE:

++         return TrackType::SubtitleStream;

++     default:

++         return std::nullopt;

++     }

++ }

++

++ namespace {

++ q23::expected<AVDemuxerContextUPtr, MediaDataHolder::ContextError>

++ loadMedia(const QUrl &mediaUrl, QIODevice *stream, const QPlaybackOptions &playbackOptions,

++           const std::shared_ptr<ICancelToken> &cancelToken)

++ {

++     using std::chrono::duration_cast;

++     using std::chrono::microseconds;

++     using std::chrono::milliseconds;

++

++     const QByteArray url = mediaUrl.toString(QUrl::PreferLocalFile).toUtf8();

++

++     AVDemuxerContextUPtr context{ avformat_alloc_context() };

++

++     if (stream) {

++         if (!stream->isOpen()) {

++             if (!stream->open(QIODevice::ReadOnly))

++                 return q23::unexpected{

++                     MediaDataHolder::ContextError{

++                             QMediaPlayer::ResourceError,

++                             QLatin1String("Could not open source device."),

++                     },

++                 };

++         }

++

++         auto seek = &seekQIODevice;

++

++         if (!stream->isSequential()) {

++             stream->seek(0);

++         } else {

++             context->ctx_flags |= AVFMTCTX_UNSEEKABLE;

++             seek = nullptr;

++         }

++

++         constexpr int bufferSize = 32768;

++         unsigned char *buffer = (unsigned char *)av_malloc(bufferSize);

++         context->pb = avio_alloc_context(buffer, bufferSize, false, stream, &readQIODevice, nullptr,

++                                          seek);

++     }

++

++     AVDictionaryHolder dict;

++     using RtmpProtocols =

++             QMinimalVarLengthFlatSet<std::basic_string_view<char16_t>, 6, std::less<>>;

++

++     static const RtmpProtocols rtmpProtocols{

++         u"rtmp", u"rtmpe", u"rtmps", u"rtmpt", u"rtmpse", u"rtmpte",

++     };

++

++     // for rtmp streams, the `timout` parameter implies acting as a server:

++     // https://ffmpeg.org/ffmpeg-protocols.html#rtmp

++     // This is not the semantics of QPlaybackOptions::networkTimeout, and will cause failures when

++     // opening streams

++     const bool setNetworkTimeout = !rtmpProtocols.contains(mediaUrl.scheme());

++

++     if (setNetworkTimeout) {

++         const milliseconds timeout = playbackOptions.networkTimeout();

++         av_dict_set_int(dict, "timeout", duration_cast<microseconds>(timeout).count(), 0);

++         qCDebug(qLcMediaDataHolder) << "Using custom network timeout:" << timeout;

++     }

++

++     {

++         const int probeSize = playbackOptions.probeSize();

++         if (probeSize != -1) {

++             constexpr int minProbeSizeFFmpeg = 32;

++             if (probeSize >= minProbeSizeFFmpeg) {

++                 av_dict_set_int(dict, "probesize", probeSize, 0);

++                 qCDebug(qLcMediaDataHolder) << "Using custom probesize" << probeSize;

++             } else

++                 qCWarning(qLcMediaDataHolder) << "Invalid probe size, using default";

++         }

++     }

++

++     const QByteArray protocolWhitelist = qgetenv("QT_FFMPEG_PROTOCOL_WHITELIST");

++     if (!protocolWhitelist.isNull())

++         av_dict_set(dict, "protocol_whitelist", protocolWhitelist.data(), 0);

++

++     if (mediaUrl.scheme().compare(u"rtsp") == 0) {

++         const QByteArray rtspTransport = qgetenv("QT_FFMPEG_RTSP_TRANSPORT").trimmed();

++         if (!rtspTransport.isEmpty()) {

++             av_dict_set(dict, "rtsp_transport", rtspTransport.constData(), 0);

++             qCDebug(qLcMediaDataHolder) << "Using custom RTSP transport:" << rtspTransport;

++         }

++     }

++

++     if (playbackOptions.playbackIntent() == QPlaybackOptions::PlaybackIntent::LowLatencyStreaming) {

++         av_dict_set(dict, "fflags", "nobuffer", 0);

++         av_dict_set_int(dict, "flush_packets", 1, 0);

++         qCDebug(qLcMediaDataHolder) << "Enabled low latency streaming";

++     }

++

++     // QTBUG-145590: for hls streams, we want to disable http persistent connections to allow FFmpeg

++     // (before FFmpeg 8?) to mix raw and encrypted streams

++     // compare  https://trac.ffmpeg.org/ticket/10599

++     if (avformat_version() < AV_VERSION_INT(62, 12, 100))

++         av_dict_set_int(dict, "http_persistent", 0, 0);

++

++ #ifdef Q_OS_ANDROID

++     // QTBUG-150720: FFmpeg >= 9.0 defaults its tls protocol to peer certificate verification,

++     // but we do not supply a CA bundle on Android by default, so every https:// source would

++     // otherwise fail to open.

++     // TODO: Apply the app's Network Security Configuration to media sources: domain-specific

++     // trust anchors, Certificate Transparency and cleartextTrafficPermitted="false". The CA

++     // bundle only carries the base-config trust anchors, so this requires verifying the peer

++     // chain per host through the same trust manager QtSslCertificates reads the anchors from

++     // (via X509TrustManagerExtensions.checkServerTrusted(chain, authType, host)), and rejecting

++     // cleartext protocols based on NetworkSecurityPolicy.isCleartextTrafficPermitted(host).

++     {

++         const QString caBundlePath = androidFFmpegCaBundlePath();

++         if (!caBundlePath.isEmpty())

++             av_dict_set(dict, "ca_file", caBundlePath.toUtf8().constData(), 0);

++     }

++ #endif

++     // NOTE: For other platforms, CA bundles in default directories are OK,

++     // and one can set non-standard locations with SSL_CERT_DIR=<directory>

++

++     context->interrupt_callback.opaque = cancelToken.get();

++     context->interrupt_callback.callback = [](void *opaque) {

++         const auto *cancelToken = static_cast<const ICancelToken *>(opaque);

++         if (cancelToken && cancelToken->isCancelled())

++             return 1;

++         return 0;

++     };

++

++     int ret = 0;

++     AVIOContext *customIo = context->pb;

++     {

++         AVFormatContext *contextRaw = context.release();

++         ret = avformat_open_input(&contextRaw, url.constData(), nullptr, dict);

++         context.reset(contextRaw);

++     }

++

++     if (ret < 0) {

++         auto code = QMediaPlayer::ResourceError;

++         if (ret == AVERROR(EACCES))

++             code = QMediaPlayer::AccessDeniedError;

++         else if (ret == AVERROR(EINVAL) || ret == AVERROR_INVALIDDATA)

++             code = QMediaPlayer::FormatError;

++

++         qCWarning(qLcMediaDataHolder)

++                 << "Could not open media. FFmpeg error description:" << AVError(ret);

++

++         freeAVIOContext(customIo);

++         return q23::unexpected{

++             MediaDataHolder::ContextError{ code, QMediaPlayer::tr("Could not open file") },

++         };

++     }

++

++     ret = avformat_find_stream_info(context.get(), nullptr);

++     if (ret < 0) {

++         return q23::unexpected{

++             MediaDataHolder::ContextError{

++                     QMediaPlayer::FormatError,

++                     QMediaPlayer::tr("Could not find stream information for media file") },

++         };

++     }

++

++     if (qLcMediaDataHolder().isInfoEnabled())

++         av_dump_format(context.get(), 0, url.constData(), 0);

++

++

++     return context;

++ }

++

++ } // namespace

++

++ MediaDataHolder::Maybe MediaDataHolder::create(const QUrl &url, QIODevice *stream,

++                                                const QPlaybackOptions &options,

++                                                const std::shared_ptr<ICancelToken> &cancelToken)

++ {

++     q23::expected context = loadMedia(url, stream, options, cancelToken);

++     if (context) {

++         // MediaDataHolder is wrapped in a shared pointer to interop with signal/slot mechanism

++         return std::make_shared<MediaDataHolder>(

++                 MediaDataHolder{ std::move(context.value()), cancelToken });

++     }

++     return q23::unexpected{ context.error() };

++ }

++

++ MediaDataHolder::MediaDataHolder(AVDemuxerContextUPtr context,

++                                  const std::shared_ptr<ICancelToken> &cancelToken)

++     : m_cancelToken{ cancelToken }

++ {

++     Q_ASSERT(context);

++

++     m_context = std::move(context);

++     m_isSeekable = !(m_context->ctx_flags & AVFMTCTX_UNSEEKABLE);

++

++     std::optional<TrackDuration> mediaDuration;

++

++     for (unsigned int i = 0; i < m_context->nb_streams; ++i) {

++

++         const auto *stream = m_context->streams[i];

++         const auto trackType = trackTypeFromMediaType(stream->codecpar->codec_type);

++

++         if (!trackType)

++             continue;

++

++         if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC)

++             continue; // Ignore attached picture streams because we treat them as metadata

++

++         if (stream->time_base.num <= 0 || stream->time_base.den <= 0) {

++             // An invalid stream timebase is not expected to be given by FFmpeg

++             qCWarning(qLcMediaDataHolder) << "A stream for the track type" << trackType

++                                           << "has an invalid timebase:" << stream->time_base;

++             continue;

++         }

++

++         auto metaData = QFFmpegMetaData::fromAVMetaData(stream->metadata);

++         const bool isDefault = stream->disposition & AV_DISPOSITION_DEFAULT;

++

++         if (trackType != TrackType::SubtitleStream) {

++             insertMediaData(metaData, *trackType, stream);

++

++             auto &requestedStream = m_requestedStreams[*trackType];

++             if (isDefault && requestedStream < 0)

++                 requestedStream = m_streamMap[*trackType].size();

++         }

++

++         if (auto duration = streamDuration(*stream))

++             metaData.insert(QMediaMetaData::Duration, toUserDuration(*duration).get());

++

++         m_streamMap[*trackType].append({ (int)i, isDefault, metaData });

++     }

++

++     // With some media files, streams may be lacking duration info. Let's

++     // get it from ffmpeg's duration estimation instead.

++     if (m_context->duration > 0ll)

++         mediaDuration = toTrackDuration(AVContextDuration(m_context->duration));

++

++     if (!mediaDuration) {

++         std::optional<AVContextDuration> contextStart = contextStartOffset(m_context.get());

++

++         std::optional<TrackPosition> startPosition = [&]() -> std::optional<TrackPosition> {

++             if (contextStart)

++                 return toTrackDuration(*contextStart).asTimePoint();

++             return std::nullopt;

++         }();

++

++         std::optional<TrackPosition> endPosition;

++

++         QSpan streams{ m_context->streams, qsizetype(m_context->nb_streams) };

++         for (const AVStream *stream : streams) {

++             std::optional<TrackPosition> currentStreamStartPosition =

++                     streamStart(*stream, m_context);

++             std::optional<TrackDuration> currentStreamDuration = streamDuration(*stream);

++             std::optional<TrackPosition> currentStreamEndPosition =

++                     [&]() -> std::optional<TrackPosition> {

++                 if (currentStreamStartPosition && currentStreamDuration)

++                     return *currentStreamStartPosition + *currentStreamDuration;

++                 return std::nullopt;

++             }();

++

++             if (startPosition)

++                 startPosition = std::min(*startPosition,

++                                          currentStreamStartPosition.value_or(*startPosition));

++             else

++                 startPosition = currentStreamStartPosition;

++

++             endPosition = std::max(endPosition, currentStreamEndPosition);

++         }

++

++         if (endPosition && startPosition)

++             mediaDuration = *endPosition - *startPosition;

++     }

++

++     m_duration = mediaDuration.value_or(TrackDuration::zero());

++

++     for (auto trackType : { TrackType::VideoStream,

++                             TrackType::AudioStream }) {

++         auto &requestedStream = m_requestedStreams[trackType];

++         auto &streamMap = m_streamMap[trackType];

++

++         if (requestedStream < 0 && !streamMap.empty())

++             requestedStream = 0;

++

++         if (requestedStream >= 0)

++             m_currentAVStreamIndex[trackType] =

++                     streamMap[requestedStream].avStreamIndex;

++     }

++

++     updateMetaData();

++ }

++

++ namespace {

++

++ /*!

++     \internal

++

++     Attempt to find an attached picture from the context's streams.

++     This will find ID3v2 pictures on audio files, and also pictures

++     attached to videos.

++  */

++ QImage getAttachedPicture(const AVFormatContext *context)

++ {

++     if (!context)

++         return {};

++

++     for (unsigned int i = 0; i < context->nb_streams; ++i) {

++         const AVStream* stream = context->streams[i];

++         if (!stream || !(stream->disposition & AV_DISPOSITION_ATTACHED_PIC))

++             continue;

++

++         const AVPacket *compressedImage = &stream->attached_pic;

++         if (!compressedImage || !compressedImage->data || compressedImage->size <= 0)

++             continue;

++

++         // Feed raw compressed data to QImage::fromData, which will decompress it

++         // if it is a recognized format.

++         QImage image = QImage::fromData({ compressedImage->data, compressedImage->size });

++         if (!image.isNull())

++             return image;

++     }

++

++     return {};

++ }

++

++ } // namespace

++

++ void MediaDataHolder::updateMetaData()

++ {

++     m_metaData = {};

++

++     if (!m_context)

++         return;

++

++     m_metaData = QFFmpegMetaData::fromAVMetaData(m_context->metadata);

++     m_metaData.insert(QMediaMetaData::FileFormat,

++                       QVariant::fromValue(QFFmpegMediaFormatInfo::fileFormatForAVInputFormat(

++                               *m_context->iformat)));

++     m_metaData.insert(QMediaMetaData::Duration, toUserDuration(m_duration).get());

++

++     if (!m_cachedThumbnail.has_value())

++         m_cachedThumbnail = getAttachedPicture(m_context.get());

++

++     QtMultimediaPrivate::setCoverArtImage(m_metaData, *m_cachedThumbnail);

++

++     for (auto trackType : { TrackType::AudioStream,

++                             TrackType::VideoStream }) {

++         const auto streamIndex = m_currentAVStreamIndex[trackType];

++         if (streamIndex >= 0)

++             insertMediaData(m_metaData, trackType, m_context->streams[streamIndex]);

++     }

++ }

++

++ bool MediaDataHolder::setActiveTrack(TrackType type, int streamNumber)

++ {

++     if (!m_context)

++         return false;

++

++     if (streamNumber < 0 || streamNumber >= m_streamMap[type].size())

++         streamNumber = -1;

++     if (m_requestedStreams[type] == streamNumber)

++         return false;

++     m_requestedStreams[type] = streamNumber;

++     const int avStreamIndex = m_streamMap[type].value(streamNumber).avStreamIndex;

++

++     const int oldIndex = m_currentAVStreamIndex[type];

++     qCDebug(qLcMediaDataHolder) << ">>>>> change track" << type << "from" << oldIndex << "to"

++                                 << avStreamIndex;

++

++     // TODO: maybe add additional verifications

++     m_currentAVStreamIndex[type] = avStreamIndex;

++

++     updateMetaData();

++

++     return true;

++ }

++

++ int MediaDataHolder::activeTrack(TrackType type) const

++ {

++     return m_requestedStreams[type];

++ }

++

++ const QList<MediaDataHolder::StreamInfo> &MediaDataHolder::streamInfo(

++         TrackType trackType) const

++ {

++     return m_streamMap[trackType];

++ }

++

++ } // namespace QFFmpeg

++

++ QT_END_NAMESPACE

Revision control