Revision control
Copy as Markdown
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegstreamdecoder_p.h"
#include "playbackengine/qffmpegmediadataholder_p.h"
#include <QtCore/qloggingcategory.h>
#include <QtCore/qregularexpression.h>
#include <QtCore/qspan.h>
QT_BEGIN_NAMESPACE
Q_STATIC_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder");
namespace QFFmpeg {
using namespace Qt::Literals;
StreamDecoder::StreamDecoder(const PlaybackEngineObjectID &id, const CodecContext &codecContext,
TrackPosition absSeekPos)
: PlaybackEngineObject(id),
m_codecContext(codecContext),
m_trackType(*MediaDataHolder::trackTypeFromMediaType(codecContext.context()->codec_type)),
m_sessionCtx{ absSeekPos }
{
qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType
<< "absSeekPos:" << absSeekPos.get();
}
StreamDecoder::~StreamDecoder()
{
avcodec_flush_buffers(m_codecContext.context());
}
void StreamDecoder::seek(quint64 sessionID, TrackPosition pos, const LoopOffset &offset)
{
updateSession(sessionID, [this, pos, offset]() {
m_sessionCtx = { offset.loopStartTimeUs.asDuration() + pos };
avcodec_flush_buffers(m_codecContext.context());
});
}
void StreamDecoder::onFinalPacketReceived(PlaybackEngineObjectID sourceID)
{
if (checkSessionID(sourceID.sessionID))
decode({});
}
void StreamDecoder::decode(Packet packet)
{
if (packet.isValid() && !checkSessionID(packet.sourceID().sessionID)) {
qCDebug(qLcStreamDecoder) << "Packet session outdated. Source id:" << packet.sourceID()
<< "current id" << id();
// no need to report packetProcessed: demuxer must be cleaned up
return;
}
m_sessionCtx.packets.enqueue(std::move(packet));
scheduleNextStep();
}
void StreamDecoder::doNextStep()
{
Packet packet = m_sessionCtx.packets.dequeue();
auto decodePacket = [this](const Packet &packet) {
if (trackType() == TrackType::SubtitleStream)
decodeSubtitle(packet);
else
decodeMedia(packet);
};
if (packet.isValid() && packet.loopOffset().loopIndex != m_sessionCtx.offset.loopIndex) {
decodePacket({});
qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:"
<< packet.loopOffset().loopIndex;
avcodec_flush_buffers(m_codecContext.context());
m_sessionCtx.offset = packet.loopOffset();
}
decodePacket(packet);
setAtEnd(!packet.isValid());
if (packet.isValid())
Q_EMIT packetProcessed(std::move(packet));
scheduleNextStep();
}
StreamDecoder::TrackType StreamDecoder::trackType() const
{
return m_trackType;
}
qint32 StreamDecoder::maxQueueSize(TrackType type)
{
switch (type) {
case TrackType::VideoStream:
return 3;
case TrackType::AudioStream:
return 9;
case TrackType::SubtitleStream:
return 6; /*main packet and closing packet*/
default:
Q_UNREACHABLE_RETURN(-1);
}
}
void StreamDecoder::onFrameProcessed(const Frame &frame)
{
if (!checkID(frame.sourceID()))
return;
--m_sessionCtx.pendingFramesCount;
Q_ASSERT(m_sessionCtx.pendingFramesCount >= 0);
scheduleNextStep();
}
bool StreamDecoder::canDoNextStep() const
{
const qint32 maxCount = maxQueueSize(m_trackType);
return !m_sessionCtx.packets.empty() && m_sessionCtx.pendingFramesCount < maxCount
&& PlaybackEngineObject::canDoNextStep();
}
void StreamDecoder::onFrameFound(const Frame &frame)
{
if (frame.isValid() && frame.absoluteEnd() < m_sessionCtx.absSeekPos)
return;
Q_ASSERT(m_sessionCtx.pendingFramesCount >= 0);
++m_sessionCtx.pendingFramesCount;
Q_EMIT requestHandleFrame(frame);
}
void StreamDecoder::decodeMedia(const Packet &packet)
{
auto sendPacketResult = sendAVPacket(packet);
if (sendPacketResult == AVERROR(EAGAIN)) {
// Doc says:
// AVERROR(EAGAIN): input is not accepted in the current state - user
// must read output with avcodec_receive_frame() (once
// all output is read, the packet should be resent, and
// the call will not fail with EAGAIN).
receiveAVFrames();
sendPacketResult = sendAVPacket(packet);
if (sendPacketResult != AVERROR(EAGAIN))
qWarning() << "Unexpected FFmpeg behavior";
}
if (sendPacketResult == 0)
receiveAVFrames(!packet.isValid());
}
int StreamDecoder::sendAVPacket(const Packet &packet)
{
return avcodec_send_packet(m_codecContext.context(), packet.isValid() ? packet.avPacket() : nullptr);
}
void StreamDecoder::receiveAVFrames(bool flushPacket)
{
while (true) {
auto avFrame = makeAVFrame();
const auto receiveFrameResult = avcodec_receive_frame(m_codecContext.context(), avFrame.get());
if (receiveFrameResult == AVERROR_EOF || receiveFrameResult == AVERROR(EAGAIN)) {
if (flushPacket && receiveFrameResult == AVERROR(EAGAIN)) {
// The documentation says that in the EAGAIN state output is not available. The new
// input must be sent. It does not say that this state can also be returned for
// Android MediaCodec when the ff_AMediaCodec_dequeueOutputBuffer call times out.
// The flush packet means it is the end of the stream. No more packets are available,
// so getting EAGAIN is unexpected here. At this point, the EAGAIN status was probably
// caused by a timeout in the ffmpeg implementation, not by too few packets. That is
// why there will be another try of calling avcodec_receive_frame
qWarning() << "Unexpected FFmpeg behavior: EAGAIN state for avcodec_receive_frame "
<< "at end of the stream";
flushPacket = false;
continue;
}
break;
}
if (receiveFrameResult < 0) {
Q_EMIT error(QMediaPlayer::FormatError, err2str(receiveFrameResult));
break;
}
// Avoid starvation on FFmpeg decoders with fixed size frame pool
if (m_trackType == TrackType::VideoStream)
avFrame = copyFromHwPool(std::move(avFrame));
onFrameFound({ m_sessionCtx.offset, std::move(avFrame), m_codecContext, id() });
}
}
void StreamDecoder::decodeSubtitle(const Packet &packet)
{
if (!packet.isValid())
return;
// qCDebug(qLcDecoder) << " decoding subtitle" << "has delay:" <<
// (codec->codec->capabilities & AV_CODEC_CAP_DELAY);
AVSubtitle subtitle;
memset(&subtitle, 0, sizeof(subtitle));
int gotSubtitle = 0;
const int res =
avcodec_decode_subtitle2(m_codecContext.context(), &subtitle, &gotSubtitle, packet.avPacket());
// qCDebug(qLcDecoder) << " subtitle got:" << res << gotSubtitle << subtitle.format <<
// Qt::hex << (quint64)subtitle.pts;
if (res < 0 || !gotSubtitle)
return;
// apparently the timestamps in the AVSubtitle structure are not always filled in
// if they are missing, use the packets pts and duration values instead
TrackPosition start = 0, end = 0;
if (subtitle.pts == AV_NOPTS_VALUE) {
start = m_codecContext.toTrackPosition(AVStreamPosition(packet.avPacket()->pts));
end = start + m_codecContext.toTrackDuration(AVStreamDuration(packet.avPacket()->duration));
} else {
auto pts = timeStampUs(subtitle.pts, AVRational{ 1, AV_TIME_BASE });
start = TrackPosition(*pts + qint64(subtitle.start_display_time) * 1000);
end = TrackPosition(*pts + qint64(subtitle.end_display_time) * 1000);
}
if (end <= start) {
qWarning() << "Invalid subtitle time";
return;
}
// qCDebug(qLcDecoder) << " got subtitle (" << start << "--" << end << "):";
QString text = subtitleTextFromAVSubtitle(subtitle);
onFrameFound({ m_sessionCtx.offset, text, start, end - start, id() });
// TODO: maybe optimize
onFrameFound({ m_sessionCtx.offset, QString(), end, TrackDuration(0), id() });
}
static QString extractSubtitleText(const AVSubtitleRect &r)
{
if (r.text)
return QString::fromUtf8(r.text);
const char *ass = r.ass;
int nCommas = 0;
while (*ass) {
if (nCommas == 8)
break;
if (*ass == ',')
++nCommas;
++ass;
}
return QString::fromUtf8(ass);
}
QString subtitleTextFromAVSubtitle(const AVSubtitle &subtitle)
{
QString text;
for (const AVSubtitleRect *r : QSpan(subtitle.rects, subtitle.num_rects)) {
if (r != subtitle.rects[0])
text += u'\n';
text += extractSubtitleText(*r);
}
static const QRegularExpression lineBreakRe(u"\\\\N|\\\\n|\\r\\n"_s);
text.replace(lineBreakRe, u"\n"_s);
if (text.endsWith(u'\n'))
text.chop(1);
return text;
}
} // namespace QFFmpeg
QT_END_NAMESPACE
#include "moc_qffmpegstreamdecoder_p.cpp"