Revision control
Copy as Markdown
// Copyright (C) 2020 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
//#define DEBUG_DECODER
#include <audio/qgstreameraudiodecoder_p.h>
#include <common/qgst_debug_p.h>
#include <common/qgstreamermessage_p.h>
#include <common/qgstutils_p.h>
#include <uri_handler/qgstreamer_qiodevice_handler_p.h>
#include <gst/gstvalue.h>
#include <gst/base/gstbasesrc.h>
#include <QtCore/qdatetime.h>
#include <QtCore/qdebug.h>
#include <QtCore/qsize.h>
#include <QtCore/qtimer.h>
#include <QtCore/qdebug.h>
#include <QtCore/qdir.h>
#include <QtCore/qstandardpaths.h>
#include <QtCore/qurl.h>
#include <QtCore/qloggingcategory.h>
QT_BEGIN_NAMESPACE
Q_STATIC_LOGGING_CATEGORY(qLcGstreamerAudioDecoder, "qt.multimedia.gstreameraudiodecoder");
typedef enum {
GST_PLAY_FLAG_VIDEO = 0x00000001,
GST_PLAY_FLAG_AUDIO = 0x00000002,
GST_PLAY_FLAG_TEXT = 0x00000004,
GST_PLAY_FLAG_VIS = 0x00000008,
GST_PLAY_FLAG_SOFT_VOLUME = 0x00000010,
GST_PLAY_FLAG_NATIVE_AUDIO = 0x00000020,
GST_PLAY_FLAG_NATIVE_VIDEO = 0x00000040,
GST_PLAY_FLAG_DOWNLOAD = 0x00000080,
GST_PLAY_FLAG_BUFFERING = 0x000000100
} GstPlayFlags;
q23::expected<QPlatformAudioDecoder *, QString> QGstreamerAudioDecoder::create(QAudioDecoder *parent)
{
static const auto error = qGstErrorMessageIfElementsNotAvailable("audioconvert", "playbin");
if (error)
return q23::unexpected{ *error };
return new QGstreamerAudioDecoder(parent);
}
QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent)
: QPlatformAudioDecoder(parent),
m_playbin{
QGstPipeline::createFromFactory("playbin3", "playbin"),
},
m_audioConvert{
QGstElement::createFromFactory("audioconvert", "audioconvert"),
}
{
// Sort out messages
m_playbin.installMessageFilter(this);
// Set the rest of the pipeline up
setAudioFlags(true);
m_outputBin = QGstBin::create("audio-output-bin");
m_outputBin.add(m_audioConvert);
// add ghostpad
m_outputBin.addGhostPad(m_audioConvert, "sink");
g_object_set(m_playbin.object(), "audio-sink", m_outputBin.element(), NULL);
// Set volume to 100%
gdouble volume = 1.0;
m_playbin.set("volume", volume);
}
QGstreamerAudioDecoder::~QGstreamerAudioDecoder()
{
stop();
m_playbin.removeMessageFilter(this);
}
bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message)
{
qCDebug(qLcGstreamerAudioDecoder) << "received bus message:" << message;
switch (message.type()) {
case GST_MESSAGE_DURATION:
return processBusMessageDuration(message);
case GST_MESSAGE_ERROR:
return processBusMessageError(message);
case GST_MESSAGE_WARNING:
return processBusMessageWarning(message);
case GST_MESSAGE_INFO:
return processBusMessageInfo(message);
case GST_MESSAGE_EOS:
return processBusMessageEOS(message);
case GST_MESSAGE_STATE_CHANGED:
return processBusMessageStateChanged(message);
case GST_MESSAGE_STREAMS_SELECTED:
return processBusMessageStreamsSelected(message);
default:
return false;
}
}
bool QGstreamerAudioDecoder::canReadQrc() const
{
return true;
}
bool QGstreamerAudioDecoder::processBusMessageError(const QGstreamerMessage &message)
{
qCDebug(qLcGstreamerAudioDecoder) << " error" << QCompactGstMessageAdaptor(message);
QUniqueGErrorHandle err;
QGString debug;
gst_message_parse_error(message.message(), &err, &debug);
if (message.source() == m_playbin) {
if (err.get()->domain == GST_STREAM_ERROR
&& err.get()->code == GST_STREAM_ERROR_CODEC_NOT_FOUND)
processInvalidMedia(QAudioDecoder::FormatError,
tr("Cannot play stream of type: <unknown>"));
else
processInvalidMedia(QAudioDecoder::ResourceError,
QString::fromUtf8(err.get()->message));
} else {
QAudioDecoder::Error qerror = QAudioDecoder::ResourceError;
if (err.get()->domain == GST_STREAM_ERROR) {
switch (err.get()->code) {
case GST_STREAM_ERROR_DECRYPT:
case GST_STREAM_ERROR_DECRYPT_NOKEY:
qerror = QAudioDecoder::AccessDeniedError;
break;
case GST_STREAM_ERROR_FORMAT:
case GST_STREAM_ERROR_DEMUX:
case GST_STREAM_ERROR_DECODE:
case GST_STREAM_ERROR_WRONG_TYPE:
case GST_STREAM_ERROR_TYPE_NOT_FOUND:
case GST_STREAM_ERROR_CODEC_NOT_FOUND:
qerror = QAudioDecoder::FormatError;
break;
default:
break;
}
} else if (err.get()->domain == GST_CORE_ERROR) {
switch (err.get()->code) {
case GST_CORE_ERROR_MISSING_PLUGIN:
qerror = QAudioDecoder::FormatError;
break;
default:
break;
}
}
processInvalidMedia(qerror, QString::fromUtf8(err.get()->message));
}
return false;
}
bool QGstreamerAudioDecoder::processBusMessageDuration(const QGstreamerMessage &)
{
updateDuration();
return false;
}
bool QGstreamerAudioDecoder::processBusMessageWarning(const QGstreamerMessage &message)
{
qCWarning(qLcGstreamerAudioDecoder) << "Warning:" << QCompactGstMessageAdaptor(message);
return false;
}
bool QGstreamerAudioDecoder::processBusMessageInfo(const QGstreamerMessage &message)
{
if (qLcGstreamerAudioDecoder().isDebugEnabled())
qCWarning(qLcGstreamerAudioDecoder) << "Info:" << QCompactGstMessageAdaptor(message);
return false;
}
bool QGstreamerAudioDecoder::processBusMessageEOS(const QGstreamerMessage &)
{
m_playbin.setState(GST_STATE_NULL);
finished();
return false;
}
bool QGstreamerAudioDecoder::processBusMessageStateChanged(const QGstreamerMessage &message)
{
if (message.source() != m_playbin)
return false;
GstState oldState;
GstState newState;
GstState pending;
gst_message_parse_state_changed(message.message(), &oldState, &newState, &pending);
bool isDecoding = false;
switch (newState) {
case GST_STATE_VOID_PENDING:
case GST_STATE_NULL:
case GST_STATE_READY:
break;
case GST_STATE_PLAYING:
isDecoding = true;
break;
case GST_STATE_PAUSED:
isDecoding = true;
// gstreamer doesn't give a reliable indication the duration
// information is ready, GST_MESSAGE_DURATION is not sent by most elements
// the duration is queried up to 5 times with increasing delay
m_durationQueries = 5;
updateDuration();
break;
}
setIsDecoding(isDecoding);
return false;
}
bool QGstreamerAudioDecoder::processBusMessageStreamsSelected(const QGstreamerMessage &message)
{
using namespace Qt::StringLiterals;
QGstStreamCollectionHandle collection;
gst_message_parse_streams_selected(const_cast<GstMessage *>(message.message()), &collection);
bool hasAudio = false;
qForeachStreamInCollection(collection, [&](GstStream *stream) {
GstStreamType type = gst_stream_get_stream_type(stream);
if (type == GstStreamType::GST_STREAM_TYPE_AUDIO)
hasAudio = true;
});
if (!hasAudio)
processInvalidMedia(QAudioDecoder::FormatError, u"No audio track in media"_s);
return false;
}
QUrl QGstreamerAudioDecoder::source() const
{
return mSource;
}
void QGstreamerAudioDecoder::setSource(const QUrl &fileName)
{
stop();
mDevice = nullptr;
bool isSignalRequired = (mSource != fileName);
mSource = fileName;
if (isSignalRequired)
sourceChanged();
}
QIODevice *QGstreamerAudioDecoder::sourceDevice() const
{
return mDevice;
}
void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device)
{
stop();
mSource.clear();
bool isSignalRequired = (mDevice != device);
mDevice = device;
if (isSignalRequired)
sourceChanged();
}
void QGstreamerAudioDecoder::start()
{
addAppSink();
if (!mSource.isEmpty()) {
m_playbin.set("uri", mSource.toEncoded().constData());
} else if (mDevice) {
// make sure we can read from device
if (!mDevice->isOpen() || !mDevice->isReadable()) {
processInvalidMedia(QAudioDecoder::ResourceError, QLatin1String("Unable to read from specified device"));
return;
}
QUrl streamURL = qGstRegisterQIODevice(mDevice);
m_playbin.set("uri", streamURL.toEncoded().constData());
} else {
return;
}
// Set audio format
if (m_appSink) {
if (mFormat.isValid()) {
setAudioFlags(false);
auto caps = QGstUtils::capsForAudioFormat(mFormat);
m_appSink.setCaps(caps);
} else {
// We want whatever the native audio format is
setAudioFlags(true);
m_appSink.setCaps({});
}
}
if (m_playbin.setState(GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
qWarning() << "GStreamer; Unable to start decoding process";
m_playbin.dumpGraph("failed");
return;
}
}
void QGstreamerAudioDecoder::stop()
{
m_playbin.setState(GST_STATE_NULL);
m_currentSessionId += 1;
removeAppSink();
// GStreamer thread is stopped. Can safely access m_buffersAvailable
if (m_buffersAvailable != 0) {
m_buffersAvailable = 0;
bufferAvailableChanged(false);
}
positionChanged(kInvalidPosition);
durationChanged(kInvalidDuration);
setIsDecoding(false);
}
QAudioFormat QGstreamerAudioDecoder::audioFormat() const
{
return mFormat;
}
void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format)
{
if (mFormat != format) {
mFormat = format;
formatChanged(mFormat);
}
}
QAudioBuffer QGstreamerAudioDecoder::read()
{
using namespace std::chrono;
QAudioBuffer audioBuffer;
if (m_buffersAvailable == 0)
return audioBuffer;
m_buffersAvailable -= 1;
if (m_buffersAvailable == 0)
bufferAvailableChanged(false);
QGstSampleHandle sample = m_appSink.pullSample();
GstBuffer *buffer = gst_sample_get_buffer(sample.get());
GstMapInfo mapInfo;
gst_buffer_map(buffer, &mapInfo, GST_MAP_READ);
const char *bufferData = (const char *)mapInfo.data;
int bufferSize = mapInfo.size;
QAudioFormat format = QGstUtils::audioFormatForSample(sample.get());
if (format.isValid()) {
// XXX At the moment we have to copy data from GstBuffer into QAudioBuffer.
// We could improve performance by implementing QAbstractAudioBuffer for GstBuffer.
nanoseconds position = getPositionFromBuffer(buffer);
// QAudioBuffer expects startTime in microseconds, but the platform
// QPlatformAudioDecoder stores/uses milliseconds resolution. Convert
// the nanoseconds position to microseconds for the buffer, and to
// milliseconds for the stored platform position.
const auto startUs = std::chrono::round<std::chrono::microseconds>(position);
audioBuffer = QAudioBuffer{ QByteArray(bufferData, bufferSize), format, startUs.count() };
positionChanged(std::chrono::round<std::chrono::milliseconds>(position));
}
gst_buffer_unmap(buffer, &mapInfo);
return audioBuffer;
}
void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString)
{
stop();
error(errorCode, errorString);
}
GstFlowReturn QGstreamerAudioDecoder::newSample(GstAppSink *)
{
// "Note that the preroll buffer will also be returned as the first buffer when calling
// gst_app_sink_pull_buffer()."
QMetaObject::invokeMethod(this, [this, sessionId = m_currentSessionId] {
if (sessionId != m_currentSessionId)
return; // stop()ed before message is executed
m_buffersAvailable += 1;
bufferAvailableChanged(true);
bufferReady();
});
return GST_FLOW_OK;
}
GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *sink, gpointer user_data)
{
QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder *>(user_data);
qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::new_sample";
return decoder->newSample(sink);
}
void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio)
{
int flags = m_playbin.getInt("flags");
// make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired
// it prevents audio format conversion
flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO);
flags |= GST_PLAY_FLAG_AUDIO;
if (wantNativeAudio)
flags |= GST_PLAY_FLAG_NATIVE_AUDIO;
m_playbin.set("flags", flags);
}
void QGstreamerAudioDecoder::addAppSink()
{
using namespace std::chrono_literals;
if (m_appSink)
return;
qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::addAppSink";
m_appSink = QGstAppSink::create("decoderAppSink");
GstAppSinkCallbacks callbacks{};
callbacks.new_sample = new_sample;
m_appSink.setCallbacks(callbacks, this, nullptr);
#if GST_CHECK_VERSION(1, 24, 0)
static constexpr auto maxBufferTime = 500ms;
m_appSink.setMaxBufferTime(maxBufferTime);
#else
static constexpr int maxBuffers = 16;
m_appSink.setMaxBuffers(maxBuffers);
#endif
static constexpr bool sync = false;
m_appSink.setSync(sync);
m_audioConvert.src().modifyPipelineInIdleProbe([&] {
m_outputBin.add(m_appSink);
qLinkGstElements(m_audioConvert, m_appSink);
});
}
void QGstreamerAudioDecoder::removeAppSink()
{
if (!m_appSink)
return;
qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::removeAppSink";
m_audioConvert.src().modifyPipelineInIdleProbe([&] {
qUnlinkGstElements(m_audioConvert, m_appSink);
m_outputBin.stopAndRemoveElements(m_appSink);
});
m_appSink = {};
}
void QGstreamerAudioDecoder::updateDuration()
{
std::optional<std::chrono::milliseconds> dur = m_playbin.durationInMs();
if (!dur)
dur = kInvalidDuration;
durationChanged(*dur);
if (dur->count() > 0)
m_durationQueries = 0;
if (m_durationQueries > 0) {
//increase delay between duration requests
int delay = 25 << (5 - m_durationQueries);
QTimer::singleShot(delay, this, &QGstreamerAudioDecoder::updateDuration);
m_durationQueries--;
}
}
std::chrono::nanoseconds QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer *buffer)
{
using namespace std::chrono;
using namespace std::chrono_literals;
nanoseconds position{ GST_BUFFER_TIMESTAMP(buffer) };
if (position >= 0ns)
return position;
else
return std::chrono::nanoseconds{ kInvalidPosition };
}
QT_END_NAMESPACE
#include "moc_qgstreameraudiodecoder_p.cpp"