Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include <common/qgstreamermediaplayer_p.h>
#include <audio/qgstreameraudiodevice_p.h>
#include <common/qglist_helper_p.h>
#include <common/qgst_debug_p.h>
#include <common/qgstutils_p.h>
#include <common/qgst_discoverer_p.h>
#include <common/qgst_play_p.h>
#include <common/qgstpipeline_p.h>
#include <common/qgstreameraudiooutput_p.h>
#include <common/qgstreamermessage_p.h>
#include <common/qgstreamermetadata_p.h>
#include <common/qgstreamervideooutput_p.h>
#include <common/qgstreamervideosink_p.h>
#include <uri_handler/qgstreamer_qiodevice_handler_p.h>
#include <qgstreamerformatinfo_p.h>
#include <QtMultimedia/private/qthreadlocalrhi_p.h>
#include <QtMultimedia/qaudiodevice.h>
#include <QtConcurrent/qtconcurrentrun.h>
#include <QtCore/qdeadlinetimer.h>
#include <QtCore/qdebug.h>
#include <QtMultimedia/private/qmultimedia_ranges_p.h>
#include <QtCore/qiodevice.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qthread.h>
#include <QtCore/qurl.h>
#include <QtCore/private/quniquehandle_p.h>
Q_STATIC_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player");
QT_BEGIN_NAMESPACE
namespace ranges = QtMultimediaPrivate::ranges;
namespace {
std::optional<QGstreamerMediaPlayer::TrackType> toTrackType(const QGstCaps &caps)
{
using TrackType = QGstreamerMediaPlayer::TrackType;
QByteArrayView type = caps.at(0).name();
if (type.startsWith("video/x-raw"))
return TrackType::VideoStream;
if (type.startsWith("audio/x-raw"))
return TrackType::AudioStream;
if (type.startsWith("text"))
return TrackType::SubtitleStream;
return std::nullopt;
}
} // namespace
QFuture<QGstreamerMediaPlayer::DiscoverResult> QGstreamerMediaPlayer::discover(QUrl url)
{
return QtConcurrent::run([url = std::move(url)] {
QGst::QGstDiscoverer discoverer;
return discoverer.discover(url);
});
}
void QGstreamerMediaPlayer::handleDiscoverResult(const DiscoverResult &discoveryResult,
const QUrl &url)
{
using namespace Qt::Literals;
using namespace std::chrono;
using namespace std::chrono_literals;
if (discoveryResult) {
// Make sure GstPlay is ready if play() is called from slots during discovery
qCDebug(qLcMediaPlayer) << "gst_play_set_uri";
gst_play_set_uri(m_gstPlay.get(), url.toEncoded().constData());
m_trackMetaData.fill({});
seekableChanged(discoveryResult->isSeekable);
if (discoveryResult->duration)
m_duration = round<milliseconds>(*discoveryResult->duration);
else
m_duration = 0ms;
durationChanged(m_duration);
m_metaData = QGst::toContainerMetadata(*discoveryResult);
videoAvailableChanged(!discoveryResult->videoStreams.empty());
audioAvailableChanged(!discoveryResult->audioStreams.empty());
m_nativeSize.clear();
for (const auto &videoInfo : discoveryResult->videoStreams) {
m_trackMetaData[0].emplace_back(QGst::toStreamMetadata(videoInfo));
QSize nativeSize = QGstUtils::qCalculateFrameSizeGStreamer(videoInfo.size,
videoInfo.pixelAspectRatio);
m_nativeSize.emplace_back(nativeSize);
}
for (const auto &audioInfo : discoveryResult->audioStreams)
m_trackMetaData[1].emplace_back(QGst::toStreamMetadata(audioInfo));
for (const auto &subtitleInfo : discoveryResult->subtitleStreams)
m_trackMetaData[2].emplace_back(QGst::toStreamMetadata(subtitleInfo));
using Key = QMediaMetaData::Key;
auto copyKeysToRootMetadata = [&](const QMediaMetaData &reference, QSpan<const Key> keys) {
for (QMediaMetaData::Key key : keys) {
QVariant referenceValue = reference.value(key);
if (referenceValue.isValid())
m_metaData.insert(key, referenceValue);
}
};
// FIXME: we duplicate some metadata for the first audio / video track
// in future we will want to use e.g. the currently selected track
if (!m_trackMetaData[0].empty())
copyKeysToRootMetadata(m_trackMetaData[0].front(),
{
Key::HasHdrContent,
Key::Orientation,
Key::Resolution,
Key::VideoBitRate,
Key::VideoCodec,
Key::VideoFrameRate,
});
if (!m_trackMetaData[1].empty())
copyKeysToRootMetadata(m_trackMetaData[1].front(),
{
Key::AudioBitRate,
Key::AudioCodec,
});
if (!m_url.isEmpty())
m_metaData.insert(QMediaMetaData::Key::Url, m_url);
qCDebug(qLcMediaPlayer) << "metadata:" << m_metaData;
qCDebug(qLcMediaPlayer) << "video metadata:" << m_trackMetaData[0];
qCDebug(qLcMediaPlayer) << "audio metadata:" << m_trackMetaData[1];
qCDebug(qLcMediaPlayer) << "subtitle metadata:" << m_trackMetaData[2];
metaDataChanged();
tracksChanged();
m_activeTrack = {
isVideoAvailable() ? 0 : -1,
isAudioAvailable() ? 0 : -1,
-1,
};
updateVideoTrackEnabled();
updateAudioTrackEnabled();
updateNativeSizeOnVideoOutput();
positionChanged(0ms);
// Handle the last play/pause/stop call made during async media loading.
m_hasPendingMedia = false;
if (m_requestedPlaybackState) {
switch (*m_requestedPlaybackState) {
case QMediaPlayer::PlayingState:
play();
break;
case QMediaPlayer::PausedState:
pause();
break;
default:
break;
}
}
} else {
qCDebug(qLcMediaPlayer) << "Discovery error:" << discoveryResult.error();
m_resourceErrorState = ResourceErrorState::ErrorOccurred;
setInvalidMediaWithError(QMediaPlayer::Error::ResourceError,
u"Resource cannot be discovered"_s);
m_hasPendingMedia = false;
resetStateForEmptyOrInvalidMedia();
};
}
void QGstreamerMediaPlayer::decoderPadAddedCustomSource(const QGstElement &src, const QGstPad &pad)
{
// GStreamer or application thread
if (src != decoder)
return;
qCDebug(qLcMediaPlayer) << "Added pad" << pad.name() << "from" << src.name();
QGstCaps caps = pad.queryCaps();
std::optional<QGstreamerMediaPlayer::TrackType> type = toTrackType(caps);
if (!type)
return;
customPipelinePads[*type] = pad;
switch (*type) {
case TrackType::VideoStream: {
QGstElement sink = gstVideoOutput->gstreamerVideoSink()
? gstVideoOutput->gstreamerVideoSink()->gstSink()
: QGstElement::createFromPipelineDescription("fakesink");
customPipeline.add(sink);
pad.link(sink.sink());
customPipelineSinks[TrackType::VideoStream] = sink;
sink.syncStateWithParent();
return;
}
case TrackType::AudioStream: {
QGstElement sink = gstAudioOutput ? gstAudioOutput->gstElement()
: QGstElement::createFromPipelineDescription("fakesink");
customPipeline.add(sink);
pad.link(sink.sink());
customPipelineSinks[TrackType::AudioStream] = sink;
sink.syncStateWithParent();
return;
}
case TrackType::SubtitleStream: {
QGstElement sink = gstVideoOutput->gstreamerVideoSink()
? gstVideoOutput->gstreamerVideoSink()->gstSink()
: QGstElement::createFromPipelineDescription("fakesink");
customPipeline.add(sink);
pad.link(sink.sink());
customPipelineSinks[TrackType::SubtitleStream] = sink;
sink.syncStateWithParent();
return;
}
default:
Q_UNREACHABLE();
}
}
void QGstreamerMediaPlayer::decoderPadRemovedCustomSource(const QGstElement &src,
const QGstPad &pad)
{
if (src != decoder)
return;
// application thread!
Q_ASSERT(thread()->isCurrentThread());
qCDebug(qLcMediaPlayer) << "Removed pad" << pad.name() << "from" << src.name() << "for stream"
<< pad.streamId();
auto found = ranges::find(customPipelinePads, pad);
if (found == customPipelinePads.end())
return;
TrackType type = static_cast<TrackType>(found - customPipelinePads.cbegin());
switch (type) {
case TrackType::VideoStream:
case TrackType::AudioStream:
case TrackType::SubtitleStream: {
if (customPipelineSinks[TrackType::VideoStream]) {
customPipeline.stopAndRemoveElements(
customPipelineSinks[TrackType::VideoStream]);
customPipelineSinks[TrackType::VideoStream] = {};
}
return;
default:
Q_UNREACHABLE();
}
}
}
void QGstreamerMediaPlayer::resetStateForEmptyOrInvalidMedia()
{
using namespace std::chrono_literals;
m_nativeSize.clear();
bool metadataNeedsSignal = !m_metaData.isEmpty();
bool tracksNeedsSignal = ranges::any_of(m_trackMetaData, [](const auto &container) {
return !container.empty();
});
m_metaData.clear();
m_trackMetaData.fill({});
m_duration = 0ms;
seekableChanged(false);
videoAvailableChanged(false);
audioAvailableChanged(false);
m_activeTrack.fill(-1);
if (metadataNeedsSignal)
metaDataChanged();
if (tracksNeedsSignal)
tracksChanged();
}
void QGstreamerMediaPlayer::updateNativeSizeOnVideoOutput()
{
int activeVideoTrack = activeTrack(TrackType::VideoStream);
bool hasVideoTrack = activeVideoTrack != -1;
QSize nativeSize = hasVideoTrack ? m_nativeSize[activeTrack(TrackType::VideoStream)] : QSize{};
QVariant orientation = hasVideoTrack
? m_trackMetaData[TrackType::VideoStream][activeVideoTrack].value(
QMediaMetaData::Key::Orientation)
: QVariant{};
if (orientation.isValid()) {
auto rotation = orientation.value<QtVideo::Rotation>();
gstVideoOutput->setRotation(rotation);
}
gstVideoOutput->setNativeSize(nativeSize);
}
void QGstreamerMediaPlayer::seekToCurrentPosition()
{
qCDebug(qLcMediaPlayer) << "gst_play_seek";
gst_play_seek(m_gstPlay.get(), gst_play_get_position(m_gstPlay.get()));
}
void QGstreamerMediaPlayer::updateVideoTrackEnabled()
{
bool hasTrack = m_activeTrack[TrackType::VideoStream] != -1;
bool hasSink = gstVideoOutput->gstreamerVideoSink() != nullptr;
gstVideoOutput->setActive(hasTrack);
gst_play_set_video_track_enabled(m_gstPlay.get(), hasTrack && hasSink);
}
void QGstreamerMediaPlayer::updateAudioTrackEnabled()
{
bool hasTrack = m_activeTrack[TrackType::AudioStream] != -1;
bool hasAudioOut = gstAudioOutput;
gst_play_set_audio_track_enabled(m_gstPlay.get(), hasTrack && hasAudioOut);
}
void QGstreamerMediaPlayer::updateBufferProgress(float newProgress)
{
if (qFuzzyIsNull(newProgress - m_bufferProgress))
return;
m_bufferProgress = newProgress;
bufferProgressChanged(m_bufferProgress);
}
void QGstreamerMediaPlayer::disconnectDecoderHandlers()
{
auto handlers = std::initializer_list<QGObjectHandlerScopedConnection *>{ &sourceSetup };
for (QGObjectHandlerScopedConnection *handler : handlers)
handler->disconnect();
}
q23::expected<QPlatformMediaPlayer *, QString> QGstreamerMediaPlayer::create(QMediaPlayer *parent)
{
auto videoOutput = QGstreamerVideoOutput::create();
if (!videoOutput)
return q23::unexpected{ videoOutput.error() };
return new QGstreamerMediaPlayer(videoOutput.value(), parent);
}
template <typename T>
void setSeekAccurate(T *config, gboolean accurate)
{
gst_play_config_set_seek_accurate(config, accurate);
}
QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput,
QMediaPlayer *parent)
: QObject(parent),
QPlatformMediaPlayer(parent),
gstVideoOutput(videoOutput),
m_gstPlay{
gst_play_new(nullptr),
QGstPlayHandle::HasRef,
},
m_playbin{
GST_PIPELINE_CAST(gst_play_get_pipeline(m_gstPlay.get())),
QGstPipeline::HasRef,
},
m_gstPlayBus{
QGstBusHandle{ gst_play_get_message_bus(m_gstPlay.get()), QGstBusHandle::HasRef },
}
{
#if 1
// LATER: remove this hack after meta-freescale decides not to pull in outdated APIs
// QTBUG-131300: nxp deliberately reverted to an old gst-play API before the gst-play API
// stabilized. compare:
// Update: Looks like first unaffected Yocto release series is 5.3 Whinlatter, so this hack can
// likely be removed when we drop Boot to Qt support for the LTS 5.0 Scarthgap (EOL April 2028).
// Link to first unaffected nxp gstreamer branch:
constexpr bool useNxpWorkaround = std::is_same_v<decltype(&gst_play_config_set_seek_accurate),
void (*)(GstPlay *, gboolean)>;
QUniqueGstStructureHandle config{
gst_play_get_config(m_gstPlay.get()),
};
if constexpr (useNxpWorkaround)
setSeekAccurate(m_gstPlay.get(), true);
else
setSeekAccurate(config.get(), true);
gst_play_set_config(m_gstPlay.get(), config.release());
#else
QUniqueGstStructureHandle config{
gst_play_get_config(m_gstPlay.get()),
};
gst_play_config_set_seek_accurate(config.get(), true);
gst_play_set_config(m_gstPlay.get(), config.release());
#endif
gstVideoOutput->setParent(this);
// NOTE: Creating a GStreamer video sink to be owned by the media player, any sink created by
// user would be a pluggable sink connected to this
m_gstVideoSink = new QGstreamerRelayVideoSink(this);
gstVideoOutput->setVideoSink(m_gstVideoSink);
m_playbin.set("video-sink", gstVideoOutput->gstElement());
m_playbin.set("text-sink", gstVideoOutput->gstSubtitleElement());
m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
m_gstPlayBus.installMessageFilter(this);
// we start without subtitles
gst_play_set_subtitle_track_enabled(m_gstPlay.get(), false);
sourceSetup = m_playbin.connect("source-setup", GCallback(sourceSetupCallback), this);
m_activeTrack.fill(-1);
// TODO: how to detect stalled media?
}
QGstreamerMediaPlayer::~QGstreamerMediaPlayer()
{
using namespace std::chrono_literals;
using namespace std::chrono;
qCDebug(qLcMediaPlayer) << "~QGstreamerMediaPlayer" << m_callStopInDestructor
<< m_expectedStoppedMessages << state();
if (customPipeline)
cleanupCustomPipeline();
m_gstPlayBus.removeMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this));
if (m_callStopInDestructor) {
m_expectedStoppedMessages += 1;
qCDebug(qLcMediaPlayer) << "gst_play_stop";
gst_play_stop(m_gstPlay.get());
}
if (m_expectedStoppedMessages) {
auto processNextMessage = [this](nanoseconds timeout) {
QGstreamerMessage message{
gst_bus_timed_pop_filtered(m_gstPlayBus.get(), timeout.count(),
GST_MESSAGE_APPLICATION), QGstreamerMessage::HasRef
};
if (!message || !gst_play_is_play_message(message.message()))
return;
GstPlayMessage type;
gst_play_message_parse_type(message.message(), &type);
qCDebug(qLcMediaPlayer) << QGstPlayMessageAdaptor{ message };
if (type == GST_PLAY_MESSAGE_END_OF_STREAM) {
m_expectedStoppedMessages += 1;
return;
}
if (type == GST_PLAY_MESSAGE_STATE_CHANGED) {
GstPlayState state;
gst_play_message_parse_state_changed(message.message(), &state);
if (state == GST_PLAY_STATE_STOPPED) {
if (m_expectedStoppedMessages > 0)
m_expectedStoppedMessages -= 1;
}
}
};
constexpr auto timeout = 200ms;
QDeadlineTimer deadline{timeout};
while (!deadline.hasExpired() && m_expectedStoppedMessages)
processNextMessage(deadline.remainingTimeAsDuration());
if (m_expectedStoppedMessages)
qWarning() << "Did not receive expected STOPPED state change messages from GstPlay "
"in QGstreamerMediaPlayer destructor:" << m_expectedStoppedMessages;
}
// NOTE: gst_play_stop is not sufficient, un-reffing m_gstPlay can deadlock
m_playbin.setStateSync(GST_STATE_NULL);
m_playbin.set("video-sink", QGstElement::createFromPipelineDescription("fakesink"));
m_playbin.set("text-sink", QGstElement::createFromPipelineDescription("fakesink"));
m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
gst_bus_set_flushing(m_gstPlayBus.get(), TRUE);
}
void QGstreamerMediaPlayer::updatePositionFromPipeline()
{
using namespace std::chrono;
positionChanged(round<milliseconds>(nanoseconds{
gst_play_get_position(m_gstPlay.get()),
}));
}
bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message)
{
if (isCustomSource()) {
constexpr bool traceBusMessages = true;
if (traceBusMessages)
qCDebug(qLcMediaPlayer) << "received bus message:" << message;
switch (message.type()) {
case GST_MESSAGE_WARNING:
qWarning() << "received bus message:" << message;
break;
case GST_MESSAGE_INFO:
qInfo() << "received bus message:" << message;
break;
case GST_MESSAGE_ERROR:
qWarning() << "received bus message:" << message;
customPipeline.dumpPipelineGraph("GST_MESSAGE_ERROR");
break;
case GST_MESSAGE_LATENCY:
customPipeline.recalculateLatency();
break;
default:
break;
}
return false;
}
switch (message.type()) {
case GST_MESSAGE_APPLICATION:
if (gst_play_is_play_message(message.message()))
return processBusMessageApplication(message);
return false;
default:
qCDebug(qLcMediaPlayer) << message;
return false;
}
return false;
}
bool QGstreamerMediaPlayer::processBusMessageApplication(const QGstreamerMessage &message)
{
using namespace std::chrono;
GstPlayMessage type;
gst_play_message_parse_type(message.message(), &type);
qCDebug(qLcMediaPlayer) << QGstPlayMessageAdaptor{ message };
switch (type) {
case GST_PLAY_MESSAGE_URI_LOADED: {
mediaStatusChanged(QMediaPlayer::LoadedMedia);
return false;
}
case GST_PLAY_MESSAGE_POSITION_UPDATED: {
if (state() == QMediaPlayer::PlaybackState::PlayingState) {
constexpr bool usePayload = false;
if constexpr (usePayload) {
GstClockTime position;
gst_play_message_parse_position_updated(message.message(), &position);
positionChanged(round<milliseconds>(nanoseconds{ position }));
} else {
GstClockTime position = gst_play_get_position(m_gstPlay.get());
positionChanged(round<milliseconds>(nanoseconds{ position }));
}
}
return false;
}
case GST_PLAY_MESSAGE_DURATION_CHANGED: {
GstClockTime duration;
gst_play_message_parse_duration_updated(message.message(), &duration);
milliseconds durationInMs = round<milliseconds>(nanoseconds{ duration });
durationChanged(durationInMs);
m_metaData.insert(QMediaMetaData::Duration, int(durationInMs.count()));
metaDataChanged();
return false;
}
case GST_PLAY_MESSAGE_BUFFERING: {
guint percent;
gst_play_message_parse_buffering_percent(message.message(), &percent);
updateBufferProgress(percent * 0.01f);
return false;
}
case GST_PLAY_MESSAGE_STATE_CHANGED: {
m_playOrPauseCalledSinceLastStateChangedOrEosMessage = false;
GstPlayState state;
gst_play_message_parse_state_changed(message.message(), &state);
switch (state) {
case GstPlayState::GST_PLAY_STATE_STOPPED:
if (m_expectedStoppedMessages > 0)
m_expectedStoppedMessages -= 1;
if (stateChangeToSkip) {
qCDebug(qLcMediaPlayer) << " skipping StoppedState transition";
stateChangeToSkip -= 1;
return false;
}
stateChanged(QMediaPlayer::StoppedState);
updateBufferProgress(0);
return false;
case GstPlayState::GST_PLAY_STATE_PAUSED:
stateChanged(QMediaPlayer::PausedState);
mediaStatusChanged(QMediaPlayer::BufferedMedia);
gstVideoOutput->setActive(true);
updateBufferProgress(1);
return false;
case GstPlayState::GST_PLAY_STATE_BUFFERING:
mediaStatusChanged(QMediaPlayer::BufferingMedia);
return false;
case GstPlayState::GST_PLAY_STATE_PLAYING:
stateChanged(QMediaPlayer::PlayingState);
mediaStatusChanged(QMediaPlayer::BufferedMedia);
gstVideoOutput->setActive(true);
updateBufferProgress(1);
return false;
default:
return false;
}
}
case GST_PLAY_MESSAGE_MEDIA_INFO_UPDATED: {
using namespace QGstPlaySupport;
QUniqueGstPlayMediaInfoHandle info{};
gst_play_message_parse_media_info_updated(message.message(), &info);
seekableChanged(gst_play_media_info_is_seekable(info.get()));
const gchar *title = gst_play_media_info_get_title(info.get());
m_metaData.insert(QMediaMetaData::Title, QString::fromUtf8(title));
metaDataChanged();
tracksChanged();
return false;
}
case GST_PLAY_MESSAGE_END_OF_STREAM: {
m_expectedStoppedMessages += 1;
// Set m_callStopInDestructor to false except in the edge case where play or pause is called
// after reaching EOS, but before we've processed this EOS message
if (!m_playOrPauseCalledSinceLastStateChangedOrEosMessage)
m_callStopInDestructor = false;
m_playOrPauseCalledSinceLastStateChangedOrEosMessage = false;
if (doLoop()) {
positionChanged(m_duration);
m_callStopInDestructor = true;
qCDebug(qLcMediaPlayer) << "EOS: restarting loop (gst_play_play)";
gst_play_play(m_gstPlay.get());
positionChanged(0ms);
// we will still get a GST_PLAY_MESSAGE_STATE_CHANGED message, which we will just ignore
// for now
stateChangeToSkip += 1;
} else {
qCDebug(qLcMediaPlayer) << "EOS: done";
positionChanged(m_duration);
mediaStatusChanged(QMediaPlayer::EndOfMedia);
stateChanged(QMediaPlayer::StoppedState);
gstVideoOutput->setActive(false);
}
return false;
}
case GST_PLAY_MESSAGE_ERROR:
case GST_PLAY_MESSAGE_WARNING:
case GST_PLAY_MESSAGE_VIDEO_DIMENSIONS_CHANGED:
case GST_PLAY_MESSAGE_VOLUME_CHANGED:
case GST_PLAY_MESSAGE_MUTE_CHANGED:
case GST_PLAY_MESSAGE_SEEK_DONE:
return false;
default:
Q_UNREACHABLE_RETURN(false);
}
}
qint64 QGstreamerMediaPlayer::duration() const
{
return m_duration.count();
}
bool QGstreamerMediaPlayer::hasMedia() const
{
return !m_url.isEmpty() || m_stream;
}
bool QGstreamerMediaPlayer::hasValidMedia() const
{
if (!hasMedia())
return false;
switch (mediaStatus()) {
case QMediaPlayer::MediaStatus::NoMedia:
case QMediaPlayer::MediaStatus::InvalidMedia:
return false;
default:
return true;
}
}
float QGstreamerMediaPlayer::bufferProgress() const
{
return m_bufferProgress;
}
QMediaTimeRange QGstreamerMediaPlayer::availablePlaybackRanges() const
{
return QMediaTimeRange();
}
qreal QGstreamerMediaPlayer::playbackRate() const
{
return gst_play_get_rate(m_gstPlay.get());
}
void QGstreamerMediaPlayer::setPlaybackRate(qreal rate)
{
if (isCustomSource()) {
static std::once_flag flag;
std::call_once(flag, [] {
// CAVEAT: unsynchronised with pipeline state. Potentially prone to race conditions
qWarning()
<< "setPlaybackRate with custom gstreamer pipelines can cause pipeline hangs. "
"Use with care";
});
customPipeline.setPlaybackRate(rate);
return;
}
if (rate == playbackRate())
return;
qCDebug(qLcMediaPlayer) << "gst_play_set_rate" << rate;
gst_play_set_rate(m_gstPlay.get(), rate);
playbackRateChanged(rate);
}
void QGstreamerMediaPlayer::setPosition(qint64 pos)
{
std::chrono::milliseconds posInMs{ pos };
setPosition(posInMs);
}
void QGstreamerMediaPlayer::setPosition(std::chrono::milliseconds pos)
{
using namespace std::chrono;
if (isCustomSource()) {
static std::once_flag flag;
std::call_once(flag, [] {
// CAVEAT: unsynchronised with pipeline state. Potentially prone to race conditions
qWarning() << "setPosition with custom gstreamer pipelines can cause pipeline hangs. "
"Use with care";
});
customPipeline.setPosition(pos);
return;
}
if (m_hasPendingMedia) {
return;
}
qCDebug(qLcMediaPlayer) << "gst_play_seek" << pos;
gst_play_seek(m_gstPlay.get(), nanoseconds(pos).count());
if (mediaStatus() == QMediaPlayer::EndOfMedia)
mediaStatusChanged(QMediaPlayer::LoadedMedia);
positionChanged(pos);
}
void QGstreamerMediaPlayer::play()
{
if (isCustomSource()) {
gstVideoOutput->setActive(true);
customPipeline.setState(GST_STATE_PLAYING);
stateChanged(QMediaPlayer::PlayingState);
return;
}
if (m_hasPendingMedia) {
// Async media loading in progress via QGstDiscoverer, m_discoveryHandler will fulfill the
// last requested playback state later.
m_requestedPlaybackState = QMediaPlayer::PlayingState;
return;
}
QMediaPlayer::PlaybackState currentState = state();
if (currentState == QMediaPlayer::PlayingState || !hasValidMedia())
return;
if (currentState != QMediaPlayer::PausedState)
resetCurrentLoop();
if (mediaStatus() == QMediaPlayer::EndOfMedia) {
positionChanged(0);
mediaStatusChanged(QMediaPlayer::LoadedMedia);
}
if (m_pendingSeek) {
qCDebug(qLcMediaPlayer) << "gst_play_seek";
gst_play_seek(m_gstPlay.get(), m_pendingSeek->count());
m_pendingSeek = std::nullopt;
}
gstVideoOutput->setActive(true);
m_callStopInDestructor = true;
m_playOrPauseCalledSinceLastStateChangedOrEosMessage = true;
qCDebug(qLcMediaPlayer) << "gst_play_play";
gst_play_play(m_gstPlay.get());
stateChanged(QMediaPlayer::PlayingState);
}
void QGstreamerMediaPlayer::pause()
{
if (isCustomSource()) {
gstVideoOutput->setActive(true);
customPipeline.setState(GST_STATE_PAUSED);
stateChanged(QMediaPlayer::PausedState);
return;
}
if (m_hasPendingMedia) {
m_requestedPlaybackState = QMediaPlayer::PausedState;
return;
}
if (state() == QMediaPlayer::PausedState || !hasMedia()
|| m_resourceErrorState != ResourceErrorState::NoError)
return;
gstVideoOutput->setActive(true);
m_callStopInDestructor = true;
m_playOrPauseCalledSinceLastStateChangedOrEosMessage = true;
qCDebug(qLcMediaPlayer) << "gst_play_pause";
gst_play_pause(m_gstPlay.get());
mediaStatusChanged(QMediaPlayer::BufferedMedia);
stateChanged(QMediaPlayer::PausedState);
}
void QGstreamerMediaPlayer::stop()
{
if (isCustomSource()) {
customPipeline.setState(GST_STATE_READY);
stateChanged(QMediaPlayer::StoppedState);
gstVideoOutput->setActive(false);
return;
}
if (m_hasPendingMedia) {
m_requestedPlaybackState = QMediaPlayer::StoppedState;
return;
}
using namespace std::chrono_literals;
if (state() == QMediaPlayer::StoppedState) {
if (position() != 0) {
m_pendingSeek = 0ms;
positionChanged(0ms);
mediaStatusChanged(QMediaPlayer::LoadedMedia);
}
return;
}
gstVideoOutput->setActive(false);
m_expectedStoppedMessages += 1;
m_callStopInDestructor = false;
qCDebug(qLcMediaPlayer) << "gst_play_stop";
gst_play_stop(m_gstPlay.get());
stateChanged(QMediaPlayer::StoppedState);
mediaStatusChanged(QMediaPlayer::LoadedMedia);
positionChanged(0ms);
}
const QGstPipeline &QGstreamerMediaPlayer::pipeline() const
{
if (isCustomSource())
return customPipeline;
return m_playbin;
}
bool QGstreamerMediaPlayer::canPlayQrc() const
{
return true;
}
bool QGstreamerMediaPlayer::pitchCompensation() const
{
return true;
}
QPlatformMediaPlayer::PitchCompensationAvailability
QGstreamerMediaPlayer::pitchCompensationAvailability() const
{
return PitchCompensationAvailability::AlwaysOn;
}
QUrl QGstreamerMediaPlayer::media() const
{
return m_url;
}
const QIODevice *QGstreamerMediaPlayer::mediaStream() const
{
return m_stream;
}
void QGstreamerMediaPlayer::sourceSetupCallback([[maybe_unused]] GstElement *playbin,
GstElement *source, QGstreamerMediaPlayer *)
{
// gst_play thread
const gchar *typeName = g_type_name_from_instance((GTypeInstance *)source);
qCDebug(qLcMediaPlayer) << "Setting up source:" << typeName;
if (typeName == std::string_view("GstRTSPSrc")) {
QGstElement s(source, QGstElement::NeedsRef);
int latency{40};
bool ok{false};
int v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_LATENCY", &ok);
if (ok)
latency = v;
qCDebug(qLcMediaPlayer) << " -> setting source latency to:" << latency << "ms";
s.set("latency", latency);
bool drop{true};
v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DROP_ON_LATENCY", &ok);
if (ok && v == 0)
drop = false;
qCDebug(qLcMediaPlayer) << " -> setting drop-on-latency to:" << drop;
s.set("drop-on-latency", drop);
bool retrans{false};
v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DO_RETRANSMISSION", &ok);
if (ok && v != 0)
retrans = true;
qCDebug(qLcMediaPlayer) << " -> setting do-retransmission to:" << retrans;
s.set("do-retransmission", retrans);
}
}
void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream)
{
if (customPipeline)
cleanupCustomPipeline();
m_resourceErrorState = ResourceErrorState::NoError;
m_url = content;
m_stream = stream;
// cancel any pending discovery continuations
m_discoveryHandler.cancel();
m_discoverFuture.cancel();
QUrl streamURL;
if (stream)
streamURL = qGstRegisterQIODevice(stream);
if (content.isEmpty() && !stream) {
mediaStatusChanged(QMediaPlayer::NoMedia);
resetStateForEmptyOrInvalidMedia();
return;
}
if (isCustomSource()) {
setMediaCustomSource(content);
} else {
mediaStatusChanged(QMediaPlayer::LoadingMedia);
m_hasPendingMedia = true;
m_requestedPlaybackState = std::nullopt;
const QUrl &playUrl = stream ? streamURL : content;
m_discoverFuture = discover(playUrl);
m_discoveryHandler =
m_discoverFuture.then(this,[this, playUrl](const DiscoverResult &result) {
handleDiscoverResult(result, playUrl);
});
}
}
void QGstreamerMediaPlayer::setMediaCustomSource(const QUrl &content)
{
using namespace Qt::Literals;
using namespace std::chrono;
using namespace std::chrono_literals;
{
// FIXME: claim sinks
// TODO: move ownership of sinks to gst_play after using them
m_playbin.set("video-sink", QGstElement::createFromPipelineDescription("fakesink"));
m_playbin.set("text-sink", QGstElement::createFromPipelineDescription("fakesink"));
m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
if (gstVideoOutput->gstreamerVideoSink()) {
if (QGstElement sink = gstVideoOutput->gstreamerVideoSink()->gstSink())
sink.removeFromParent();
}
}
customPipeline = QGstPipeline::create("customPipeline");
customPipeline.installMessageFilter(this);
positionUpdateTimer = std::make_unique<QTimer>();
QObject::connect(positionUpdateTimer.get(), &QTimer::timeout, this, [this] {
Q_ASSERT(customPipeline);
auto position = customPipeline.position();
positionChanged(round<milliseconds>(position));
});
positionUpdateTimer->start(100ms);
QByteArray gstLaunchString =
content.toString(QUrl::RemoveScheme | QUrl::PrettyDecoded).toLatin1();
qCDebug(qLcMediaPlayer) << "generating" << gstLaunchString;
QGstElement element = QGstElement::createFromPipelineDescription(gstLaunchString);
if (!element) {
Q_EMIT error(QMediaPlayer::ResourceError, u"Could not create custom pipeline"_s);
return;
}
decoder = element;
customPipeline.add(decoder);
QGstBin elementBin{
qGstSafeCast<GstBin>(element.element()),
QGstBin::NeedsRef,
};
if (elementBin) // bins are expected to provide unconnected src pads
elementBin.addUnlinkedGhostPads(GstPadDirection::GST_PAD_SRC);
// for all other elements
padAdded = decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAddedCustomSource>(this);
padRemoved = decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemovedCustomSource>(this);
customPipeline.setStateSync(GstState::GST_STATE_PAUSED);
auto srcPadVisitor = [](GstElement *element, GstPad *pad, void *self) -> gboolean {
reinterpret_cast<QGstreamerMediaPlayer *>(self)->decoderPadAddedCustomSource(
QGstElement{ element, QGstElement::NeedsRef }, QGstPad{ pad, QGstPad::NeedsRef });
return true;
};
gst_element_foreach_pad(element.element(), srcPadVisitor, this);
mediaStatusChanged(QMediaPlayer::LoadedMedia);
customPipeline.dumpGraph("setMediaCustomPipeline");
}
void QGstreamerMediaPlayer::cleanupCustomPipeline()
{
customPipeline.setStateSync(GST_STATE_NULL);
customPipeline.removeMessageFilter(this);
for (QGstElement &sink : customPipelineSinks)
if (sink)
customPipeline.remove(sink);
positionUpdateTimer = {};
customPipeline = {};
}
void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output)
{
if (isCustomSource()) {
qWarning() << "QMediaPlayer::setAudioOutput not supported when using custom sources";
return;
}
if (gstAudioOutput == output)
return;
auto *gstOutput = static_cast<QGstreamerAudioOutput *>(output);
if (gstOutput)
gstOutput->setAsync(true);
gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output);
if (gstAudioOutput)
m_playbin.set("audio-sink", gstAudioOutput->gstElement());
else
m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
updateAudioTrackEnabled();
// FIXME: we need to have a gst_play API to change the sinks on the fly.
// finishStateChange a hack to avoid assertion failures in gstreamer
if (!qmediaplayerDestructorCalled)
m_playbin.finishStateChange();
}
QMediaMetaData QGstreamerMediaPlayer::metaData() const
{
return m_metaData;
}
void QGstreamerMediaPlayer::setVideoSink(QVideoSink *sink)
{
// Disconnect previous sink
m_gstVideoSink->disconnectPluggableVideoSink();
if (!sink)
return;
// Connect pluggable sink to native sink
auto pluggableSink = dynamic_cast<QGstreamerPluggableVideoSink *>(sink->platformVideoSink());
Q_ASSERT(pluggableSink);
m_gstVideoSink->connectPluggableVideoSink(pluggableSink);
}
int QGstreamerMediaPlayer::trackCount(TrackType type)
{
QSpan<const QMediaMetaData> tracks = m_trackMetaData[type];
return tracks.size();
}
QMediaMetaData QGstreamerMediaPlayer::trackMetaData(TrackType type, int index)
{
QSpan<const QMediaMetaData> tracks = m_trackMetaData[type];
if (index < tracks.size())
return tracks[index];
return {};
}
int QGstreamerMediaPlayer::activeTrack(TrackType type)
{
return m_activeTrack[type];
}
void QGstreamerMediaPlayer::setActiveTrack(TrackType type, int index)
{
if (m_activeTrack[type] == index)
return;
int formerTrack = m_activeTrack[type];
m_activeTrack[type] = index;
switch (type) {
case TrackType::VideoStream: {
if (index != -1)
gst_play_set_video_track(m_gstPlay.get(), index);
updateVideoTrackEnabled();
updateNativeSizeOnVideoOutput();
break;
}
case TrackType::AudioStream: {
if (index != -1)
gst_play_set_audio_track(m_gstPlay.get(), index);
updateAudioTrackEnabled();
break;
}
case TrackType::SubtitleStream: {
if (index != -1)
gst_play_set_subtitle_track(m_gstPlay.get(), index);
gst_play_set_subtitle_track_enabled(m_gstPlay.get(), index != -1);
break;
}
default:
Q_UNREACHABLE();
};
if (formerTrack != -1 && index != -1)
// it can take several seconds for gstreamer to switch the track. so we seek to the current
// position
seekToCurrentPosition();
}
QT_END_NAMESPACE