Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies).
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "avfvideorenderercontrol_p.h"
#include <avfvideobuffer_p.h>
#include <avfvideosink_p.h>
#include <mediaplayer/avfdisplaylink_p.h>
#include <QtMultimedia/qvideoframeformat.h>
#include <QtMultimedia/private/qavfhelpers_p.h>
#include <QtMultimedia/private/qvideoframe_p.h>
#include <QtGui/rhi/qrhi.h>
#include <QtCore/qdebug.h>
#import <AVFoundation/AVFoundation.h>
#include <CoreVideo/CVPixelBuffer.h>
#include <CoreVideo/CVImageBuffer.h>
QT_USE_NAMESPACE
@interface SubtitleDelegate : NSObject <AVPlayerItemLegibleOutputPushDelegate>
{
AVFVideoRendererControl *m_renderer;
}
- (void)legibleOutput:(AVPlayerItemLegibleOutput *)output
didOutputAttributedStrings:(NSArray<NSAttributedString *> *)strings
nativeSampleBuffers:(NSArray *)nativeSamples
forItemTime:(CMTime)itemTime;
@end
@implementation SubtitleDelegate
-(id)initWithRenderer: (AVFVideoRendererControl *)renderer
{
if (!(self = [super init]))
return nil;
m_renderer = renderer;
return self;
}
- (void)legibleOutput:(AVPlayerItemLegibleOutput *)output
didOutputAttributedStrings:(NSArray<NSAttributedString *> *)strings
nativeSampleBuffers:(NSArray *)nativeSamples
forItemTime:(CMTime)itemTime
{
QString text;
for (NSAttributedString *s : strings) {
if (!text.isEmpty())
text += QChar::LineSeparator;
text += QString::fromNSString(s.string);
}
m_renderer->setSubtitleText(text);
}
@end
AVFVideoRendererControl::AVFVideoRendererControl(QObject *parent)
: QObject(parent)
{
m_displayLink = new AVFDisplayLink(this);
connect(m_displayLink, &AVFDisplayLink::tick, this, &AVFVideoRendererControl::updateVideoFrame);
}
AVFVideoRendererControl::~AVFVideoRendererControl()
{
#ifdef QT_DEBUG_AVF
qDebug() << Q_FUNC_INFO;
#endif
m_displayLink->stop();
m_displayLink->disconnect(this);
if (m_videoOutput)
[m_videoOutput release];
if (m_subtitleOutput)
[m_subtitleOutput release];
if (m_subtitleDelegate)
[m_subtitleDelegate release];
}
void AVFVideoRendererControl::reconfigure()
{
#ifdef QT_DEBUG_AVF
qDebug() << "reconfigure";
#endif
if (!m_layer) {
m_displayLink->stop();
return;
}
QMutexLocker locker(&m_mutex);
m_displayLink->start();
nativeSizeChanged();
}
void AVFVideoRendererControl::setLayer(CALayer *layer)
{
if (m_layer == layer)
return;
AVPlayerLayer *plLayer = playerLayer();
if (plLayer) {
if (m_videoOutput)
[[[plLayer player] currentItem] removeOutput:m_videoOutput];
if (m_subtitleOutput)
[[[plLayer player] currentItem] removeOutput:m_subtitleOutput];
}
if (!layer && m_sink)
m_sink->setVideoFrame(QVideoFrame());
AVFVideoSinkInterface::setLayer(layer);
}
void AVFVideoRendererControl::setVideoRotation(QtVideo::Rotation rotation)
{
m_rotation = rotation;
}
void AVFVideoRendererControl::setVideoMirrored(bool mirrored)
{
m_mirrored = mirrored;
}
void AVFVideoRendererControl::updateVideoFrame()
{
if (!m_sink)
return;
if (!m_layer)
return;
auto *layer = playerLayer();
if (!layer.readyForDisplay)
return;
nativeSizeChanged();
QVideoFrame frame;
size_t width, height;
qint64 startTime = -1;
QCFType<CVPixelBufferRef> pixelBuffer = copyPixelBufferFromLayer(width, height, startTime);
if (!pixelBuffer)
return;
// qDebug() << "Got pixelbuffer with format" << fmt << Qt::hex <<
// CVPixelBufferGetPixelFormatType(pixelBuffer);
auto buffer = std::make_unique<AVFVideoBuffer>(this, std::move(pixelBuffer));
auto format = buffer->videoFormat();
format.setRotation(m_rotation);
format.setMirrored(m_mirrored);
frame = QVideoFramePrivate::createFrame(std::move(buffer), format);
if (startTime >= 0) {
frame.setStartTime(startTime);
// Estimate end time from video track's nominal frame rate
AVPlayerItem *playerItem = [layer.player currentItem];
if (playerItem) {
float fps = 0;
for (AVPlayerItemTrack *track in playerItem.tracks) {
if ([track.assetTrack.mediaType isEqualToString:AVMediaTypeVideo]) {
fps = track.assetTrack.nominalFrameRate;
break;
}
}
if (fps > 0)
frame.setEndTime(startTime + qint64(1000000.0 / fps));
}
}
m_sink->setVideoFrame(frame);
}
QCFType<CVPixelBufferRef>
AVFVideoRendererControl::copyPixelBufferFromLayer(size_t &width, size_t &height, qint64 &startTime)
{
AVPlayerLayer *layer = playerLayer();
//Is layer valid
if (!layer) {
#ifdef QT_DEBUG_AVF
qWarning("copyPixelBufferFromLayer: invalid layer");
#endif
return nullptr;
}
AVPlayerItem * item = [[layer player] currentItem];
if (!m_videoOutput) {
if (!m_outputSettings)
setOutputSettings();
m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:m_outputSettings];
[m_videoOutput setDelegate:nil queue:nil];
}
if (!m_subtitleOutput) {
m_subtitleOutput = [[AVPlayerItemLegibleOutput alloc] init];
m_subtitleDelegate = [[SubtitleDelegate alloc] initWithRenderer:this];
[m_subtitleOutput setDelegate:m_subtitleDelegate queue:dispatch_get_main_queue()];
}
if (![item.outputs containsObject:m_videoOutput])
[item addOutput:m_videoOutput];
if (![item.outputs containsObject:m_subtitleOutput])
[item addOutput:m_subtitleOutput];
CFTimeInterval currentCAFrameTime = CACurrentMediaTime();
CMTime currentCMFrameTime = [m_videoOutput itemTimeForHostTime:currentCAFrameTime];
// happens when buffering / loading
if (CMTimeCompare(currentCMFrameTime, kCMTimeZero) < 0) {
return nullptr;
}
if (![m_videoOutput hasNewPixelBufferForItemTime:currentCMFrameTime])
return nullptr;
CMTime outItemTimeForDisplay = kCMTimeInvalid;
CVPixelBufferRef pixelBuffer =
[m_videoOutput copyPixelBufferForItemTime:currentCMFrameTime
itemTimeForDisplay:&outItemTimeForDisplay];
if (!pixelBuffer) {
#ifdef QT_DEBUG_AVF
qWarning("copyPixelBufferForItemTime returned nil");
CMTimeShow(currentCMFrameTime);
#endif
return nullptr;
}
CMTime pts =
CMTIME_IS_VALID(outItemTimeForDisplay) ? outItemTimeForDisplay : currentCMFrameTime;
startTime = qint64(qreal(pts.value) / pts.timescale * 1000000.0);
width = CVPixelBufferGetWidth(pixelBuffer);
height = CVPixelBufferGetHeight(pixelBuffer);
// auto f = CVPixelBufferGetPixelFormatType(pixelBuffer);
// char fmt[5];
// memcpy(fmt, &f, 4);
// fmt[4] = 0;
// qDebug() << "copyPixelBuffer" << f << fmt << width << height;
return QCFType<CVPixelBufferRef>{ pixelBuffer };
}
#include "moc_avfvideorenderercontrol_p.cpp"