Revision control
++ // Copyright (C) 2021 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++ #include "qffmpegstreamdecoder_p.h"
++
++ #include "playbackengine/qffmpegmediadataholder_p.h"
++ #include <QtCore/qloggingcategory.h>
++ #include <QtCore/qregularexpression.h>
++ #include <QtCore/qspan.h>
++
++ QT_BEGIN_NAMESPACE
++
++ Q_STATIC_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder");
++
++ namespace QFFmpeg {
++
++ using namespace Qt::Literals;
++
++ StreamDecoder::StreamDecoder(const PlaybackEngineObjectID &id, const CodecContext &codecContext,
++ TrackPosition absSeekPos)
++ : PlaybackEngineObject(id),
++ m_codecContext(codecContext),
++ m_trackType(*MediaDataHolder::trackTypeFromMediaType(codecContext.context()->codec_type)),
++ m_sessionCtx{ absSeekPos }
++ {
++ qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType
++ << "absSeekPos:" << absSeekPos.get();
++ }
++
++ StreamDecoder::~StreamDecoder()
++ {
++ avcodec_flush_buffers(m_codecContext.context());
++ }
++
++ void StreamDecoder::seek(quint64 sessionID, TrackPosition pos, const LoopOffset &offset)
++ {
++ updateSession(sessionID, [this, pos, offset]() {
++ m_sessionCtx = { offset.loopStartTimeUs.asDuration() + pos };
++ avcodec_flush_buffers(m_codecContext.context());
++ });
++ }
++
++ void StreamDecoder::onFinalPacketReceived(PlaybackEngineObjectID sourceID)
++ {
++ if (checkSessionID(sourceID.sessionID))
++ decode({});
++ }
++
++ void StreamDecoder::decode(Packet packet)
++ {
++ if (packet.isValid() && !checkSessionID(packet.sourceID().sessionID)) {
++ qCDebug(qLcStreamDecoder) << "Packet session outdated. Source id:" << packet.sourceID()
++ << "current id" << id();
++ // no need to report packetProcessed: demuxer must be cleaned up
++ return;
++ }
++
++ m_sessionCtx.packets.enqueue(std::move(packet));
++ scheduleNextStep();
++ }
++
++ void StreamDecoder::doNextStep()
++ {
++ Packet packet = m_sessionCtx.packets.dequeue();
++
++ auto decodePacket = [this](const Packet &packet) {
++ if (trackType() == TrackType::SubtitleStream)
++ decodeSubtitle(packet);
++ else
++ decodeMedia(packet);
++ };
++
++ if (packet.isValid() && packet.loopOffset().loopIndex != m_sessionCtx.offset.loopIndex) {
++ decodePacket({});
++
++ qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:"
++ << packet.loopOffset().loopIndex;
++
++ avcodec_flush_buffers(m_codecContext.context());
++ m_sessionCtx.offset = packet.loopOffset();
++ }
++
++ decodePacket(packet);
++
++ setAtEnd(!packet.isValid());
++
++ if (packet.isValid())
++ Q_EMIT packetProcessed(std::move(packet));
++
++ scheduleNextStep();
++ }
++
++ StreamDecoder::TrackType StreamDecoder::trackType() const
++ {
++ return m_trackType;
++ }
++
++ qint32 StreamDecoder::maxQueueSize(TrackType type)
++ {
++ switch (type) {
++
++ case TrackType::VideoStream:
++ return 3;
++ case TrackType::AudioStream:
++ return 9;
++ case TrackType::SubtitleStream:
++ return 6; /*main packet and closing packet*/
++ default:
++ Q_UNREACHABLE_RETURN(-1);
++ }
++ }
++
++ void StreamDecoder::onFrameProcessed(const Frame &frame)
++ {
++ if (!checkID(frame.sourceID()))
++ return;
++
++ --m_sessionCtx.pendingFramesCount;
++ Q_ASSERT(m_sessionCtx.pendingFramesCount >= 0);
++
++ scheduleNextStep();
++ }
++
++ bool StreamDecoder::canDoNextStep() const
++ {
++ const qint32 maxCount = maxQueueSize(m_trackType);
++
++ return !m_sessionCtx.packets.empty() && m_sessionCtx.pendingFramesCount < maxCount
++ && PlaybackEngineObject::canDoNextStep();
++ }
++
++ void StreamDecoder::onFrameFound(const Frame &frame)
++ {
++ if (frame.isValid() && frame.absoluteEnd() < m_sessionCtx.absSeekPos)
++ return;
++
++ Q_ASSERT(m_sessionCtx.pendingFramesCount >= 0);
++ ++m_sessionCtx.pendingFramesCount;
++ Q_EMIT requestHandleFrame(frame);
++ }
++
++ void StreamDecoder::decodeMedia(const Packet &packet)
++ {
++ auto sendPacketResult = sendAVPacket(packet);
++
++ if (sendPacketResult == AVERROR(EAGAIN)) {
++ // Doc says:
++ // AVERROR(EAGAIN): input is not accepted in the current state - user
++ // must read output with avcodec_receive_frame() (once
++ // all output is read, the packet should be resent, and
++ // the call will not fail with EAGAIN).
++ receiveAVFrames();
++ sendPacketResult = sendAVPacket(packet);
++
++ if (sendPacketResult != AVERROR(EAGAIN))
++ qWarning() << "Unexpected FFmpeg behavior";
++ }
++
++ if (sendPacketResult == 0)
++ receiveAVFrames(!packet.isValid());
++ }
++
++ int StreamDecoder::sendAVPacket(const Packet &packet)
++ {
++ return avcodec_send_packet(m_codecContext.context(), packet.isValid() ? packet.avPacket() : nullptr);
++ }
++
++ void StreamDecoder::receiveAVFrames(bool flushPacket)
++ {
++ while (true) {
++ auto avFrame = makeAVFrame();
++
++ const auto receiveFrameResult = avcodec_receive_frame(m_codecContext.context(), avFrame.get());
++
++ if (receiveFrameResult == AVERROR_EOF || receiveFrameResult == AVERROR(EAGAIN)) {
++ if (flushPacket && receiveFrameResult == AVERROR(EAGAIN)) {
++ // The documentation says that in the EAGAIN state output is not available. The new
++ // input must be sent. It does not say that this state can also be returned for
++ // Android MediaCodec when the ff_AMediaCodec_dequeueOutputBuffer call times out.
++ // The flush packet means it is the end of the stream. No more packets are available,
++ // so getting EAGAIN is unexpected here. At this point, the EAGAIN status was probably
++ // caused by a timeout in the ffmpeg implementation, not by too few packets. That is
++ // why there will be another try of calling avcodec_receive_frame
++ qWarning() << "Unexpected FFmpeg behavior: EAGAIN state for avcodec_receive_frame "
++ << "at end of the stream";
++ flushPacket = false;
++ continue;
++ }
++ break;
++ }
++
++ if (receiveFrameResult < 0) {
++ Q_EMIT error(QMediaPlayer::FormatError, err2str(receiveFrameResult));
++ break;
++ }
++
++
++ // Avoid starvation on FFmpeg decoders with fixed size frame pool
++ if (m_trackType == TrackType::VideoStream)
++ avFrame = copyFromHwPool(std::move(avFrame));
++
++ onFrameFound({ m_sessionCtx.offset, std::move(avFrame), m_codecContext, id() });
++ }
++ }
++
++ void StreamDecoder::decodeSubtitle(const Packet &packet)
++ {
++ if (!packet.isValid())
++ return;
++ // qCDebug(qLcDecoder) << " decoding subtitle" << "has delay:" <<
++ // (codec->codec->capabilities & AV_CODEC_CAP_DELAY);
++ AVSubtitle subtitle;
++ memset(&subtitle, 0, sizeof(subtitle));
++ int gotSubtitle = 0;
++
++ const int res =
++ avcodec_decode_subtitle2(m_codecContext.context(), &subtitle, &gotSubtitle, packet.avPacket());
++ // qCDebug(qLcDecoder) << " subtitle got:" << res << gotSubtitle << subtitle.format <<
++ // Qt::hex << (quint64)subtitle.pts;
++ if (res < 0 || !gotSubtitle)
++ return;
++
++ // apparently the timestamps in the AVSubtitle structure are not always filled in
++ // if they are missing, use the packets pts and duration values instead
++ TrackPosition start = 0, end = 0;
++ if (subtitle.pts == AV_NOPTS_VALUE) {
++ start = m_codecContext.toTrackPosition(AVStreamPosition(packet.avPacket()->pts));
++ end = start + m_codecContext.toTrackDuration(AVStreamDuration(packet.avPacket()->duration));
++ } else {
++ auto pts = timeStampUs(subtitle.pts, AVRational{ 1, AV_TIME_BASE });
++ start = TrackPosition(*pts + qint64(subtitle.start_display_time) * 1000);
++ end = TrackPosition(*pts + qint64(subtitle.end_display_time) * 1000);
++ }
++
++ if (end <= start) {
++ qWarning() << "Invalid subtitle time";
++ return;
++ }
++ // qCDebug(qLcDecoder) << " got subtitle (" << start << "--" << end << "):";
++ QString text = subtitleTextFromAVSubtitle(subtitle);
++
++ onFrameFound({ m_sessionCtx.offset, text, start, end - start, id() });
++
++ // TODO: maybe optimize
++ onFrameFound({ m_sessionCtx.offset, QString(), end, TrackDuration(0), id() });
++ }
++
++ static QString extractSubtitleText(const AVSubtitleRect &r)
++ {
++ if (r.text)
++ return QString::fromUtf8(r.text);
++
++ const char *ass = r.ass;
++ int nCommas = 0;
++ while (*ass) {
++ if (nCommas == 8)
++ break;
++ if (*ass == ',')
++ ++nCommas;
++ ++ass;
++ }
++ return QString::fromUtf8(ass);
++ }
++
++ QString subtitleTextFromAVSubtitle(const AVSubtitle &subtitle)
++ {
++ QString text;
++
++ for (const AVSubtitleRect *r : QSpan(subtitle.rects, subtitle.num_rects)) {
++ if (r != subtitle.rects[0])
++ text += u'\n';
++
++ text += extractSubtitleText(*r);
++ }
++ static const QRegularExpression lineBreakRe(u"\\\\N|\\\\n|\\r\\n"_s);
++ text.replace(lineBreakRe, u"\n"_s);
++ if (text.endsWith(u'\n'))
++ text.chop(1);
++ return text;
++ }
++
++ } // namespace QFFmpeg
++
++ QT_END_NAMESPACE
++
++ #include "moc_qffmpegstreamdecoder_p.cpp"