qt/qtmultimedia/tests/auto/integration/qmediaframeinputsbackend/tst_qmediaframeinputsbackend.cpp (file symbol)

Source code

Revision control

Copy as Markdown

// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
#include "tst_qmediaframeinputsbackend.h"
#include <QtMultimediaTestLib/private/audiogenerationutils_p.h>
#include <QtMultimediaTestLib/private/capturesessionfixture_p.h>
#include <QtMultimediaTestLib/private/mediabackendutils_p.h>
#include <QtMultimediaTestLib/private/mediainfo_p.h>
#include <QtMultimediaTestLib/private/qcolorutil_p.h>
#include <QtMultimediaTestLib/private/testvideosink_p.h>
#include <QtTest/qsignalspy.h>
#include <QtTest/qtest.h>
#include <QtMultimedia/qaudiobufferinput.h>
#include <QtMultimedia/qmediaplayer.h>
#include <QtMultimedia/qmediarecorder.h>
#include <QtMultimedia/qvideoframeinput.h>
#include <QtMultimedia/private/qplatformaudioresampler_p.h>
#include <QtMultimedia/private/qplatformmediaintegration_p.h>
#include <thread>
QT_BEGIN_NAMESPACE
namespace {
struct AudioComparisonResult
{
struct ChannelInfo
{
qreal normalizedCrossCorrelation = 0;
qreal maxDeviation = 0;
qreal avgDeviation = 0;
};
size_t actualSampleCount = 0;
size_t expectedSampleCount = 0;
size_t actualSamplesOffset = 0;
std::vector<ChannelInfo> channelsInfo;
bool check() const
{
return std::all_of(channelsInfo.begin(), channelsInfo.end(), [](const ChannelInfo &info) {
return info.normalizedCrossCorrelation > 0.96;
});
}
QString toString() const
{
QString result;
QTextStream stream(&result);
stream << "AudioComparisonResult:";
stream << "\n\tactualSampleCount: " << actualSampleCount;
stream << "\n\texpectedSampleCount: " << expectedSampleCount;
stream << "\n\tactualSamplesOffset: " << actualSamplesOffset;
int channel = 0;
for (auto &chInfo : channelsInfo) {
stream << "\n\tchannel: " << channel++;
stream << "\n\t\tnormalizedCrossCorrelation: " << chInfo.normalizedCrossCorrelation;
stream << "\n\t\tmaxDeviation: " << chInfo.maxDeviation;
stream << "\n\t\tavgDeviation: " << chInfo.avgDeviation;
}
stream.flush();
return result;
}
};
AudioComparisonResult::ChannelInfo compareChannelAudioData(const float *lhs, const float *rhs,
quint32 samplesCount, int channel,
int channelsCount)
{
AudioComparisonResult::ChannelInfo result;
qreal crossCorrelation = 0.;
qreal lhsStandardDeviation = 0.;
qreal rhsStandardDeviation = 0.;
qreal deviationsSum = 0.;
size_t i = channel;
for (quint32 sample = 0; sample < samplesCount; ++sample, i += channelsCount) {
crossCorrelation += lhs[i] * rhs[i];
lhsStandardDeviation += lhs[i] * lhs[i];
rhsStandardDeviation += rhs[i] * rhs[i];
const qreal deviation = qAbs(lhs[i] - rhs[i]);
deviationsSum += deviation;
result.maxDeviation = qMax(result.maxDeviation, deviation);
}
lhsStandardDeviation = sqrt(lhsStandardDeviation);
rhsStandardDeviation = sqrt(rhsStandardDeviation);
result.normalizedCrossCorrelation =
crossCorrelation / (lhsStandardDeviation * rhsStandardDeviation);
result.avgDeviation = deviationsSum / samplesCount;
return result;
}
AudioComparisonResult compareAudioData(QSpan<const float> actual, QSpan<const float> expected,
int channelsCount)
{
AudioComparisonResult result;
result.actualSampleCount = actual.size() / channelsCount;
result.expectedSampleCount = expected.size() / channelsCount;
// can be calculated
result.actualSamplesOffset = 0;
const quint32 samplesCount = static_cast<quint32>(qMin(
result.actualSampleCount - result.actualSamplesOffset, result.expectedSampleCount));
for (int channel = 0; channel < channelsCount; ++channel)
result.channelsInfo.push_back(
compareChannelAudioData(actual.data() + result.actualSamplesOffset * channelsCount,
expected.data(), samplesCount, channel, channelsCount));
return result;
}
QAudioBuffer convertAudioBuffer(QAudioBuffer buffer, const QAudioFormat &format)
{
if (format == buffer.format())
return buffer;
auto resampler =
QPlatformMediaIntegration::instance()->createAudioResampler(buffer.format(), format);
if (!resampler)
return {};
return resampler.value()->resample(buffer.constData<char>(), buffer.byteCount());
}
QSpan<const float> toFloatSpan(const QAudioBuffer &buffer)
{
return QSpan(buffer.constData<const float>(), buffer.byteCount() / sizeof(float));
}
AudioComparisonResult compareAudioData(QAudioBuffer actual, QAudioBuffer expected)
{
QAudioFormat format = actual.format();
format.setSampleFormat(QAudioFormat::Float);
actual = convertAudioBuffer(std::move(actual), format);
expected = convertAudioBuffer(std::move(expected), format);
return compareAudioData(toFloatSpan(actual), toFloatSpan(expected), format.channelCount());
}
} // namespace
void tst_QMediaFrameInputsBackend::initTestCase()
{
if (!initIntegrationTestCase())
return;
QSKIP_WMF("QVideoFrameInput/QAudioBufferInput not supported");
QSKIP_GSTREAMER("Not implemented in the gstreamer backend");
QSKIP_OHOS("OHOS backend does not yet wire QPlatformAudioBufferInput/QPlatformVideoFrameInput into a muxer-based recorder");
}
void tst_QMediaFrameInputsBackend::mediaRecorderWritesAudio_whenAudioFramesInputSends_data()
{
QTest::addColumn<int>("bufferCount");
QTest::addColumn<QAudioFormat::SampleFormat>("sampleFormat");
QTest::addColumn<QAudioFormat::ChannelConfig>("channelConfig");
QTest::addColumn<int>("sampleRate");
QTest::addColumn<milliseconds>("duration");
#ifndef Q_OS_WINDOWS // sample rate 8000 is not supported. TODO: investigate.
QTest::addRow("bufferCount: 20; sampleFormat: Int16; channelConfig: Mono; sampleRate: 8000; "
"duration: 1000")
<< 20 << QAudioFormat::Int16 << QAudioFormat::ChannelConfigMono << 8000 << 1000ms;
#endif
QTest::addRow("bufferCount: 30; sampleFormat: Int32; channelConfig: Stereo; sampleRate: "
"16000; duration: 2000")
<< 30 << QAudioFormat::Int32 << QAudioFormat::ChannelConfigStereo << 16000 << 2000ms;
QTest::addRow("bufferCount: 30; sampleFormat: Int16; channelConfig: Mono; sampleRate: "
"44100; duration: 2000")
<< 30 << QAudioFormat::Int16 << QAudioFormat::ChannelConfigMono << 44100 << 2000ms;
// TODO: investigate fails of channels configuration
// QTest::addRow("bufferCount: 10; sampleFormat: UInt8; channelConfig: 2Dot1; sampleRate:
// 40000; duration: 1500")
// << 10 << QAudioFormat::UInt8 << QAudioFormat::ChannelConfig2Dot1 << 40000 << 1500;
// QTest::addRow("bufferCount: 10; sampleFormat: Float; channelConfig: 3Dot0; sampleRate:
// 50000; duration: 2500")
// << 40 << QAudioFormat::Float << QAudioFormat::ChannelConfig3Dot0 << 50000 << 2500;
}
void tst_QMediaFrameInputsBackend::mediaRecorderWritesAudio_whenAudioFramesInputSends()
{
QFETCH(const int, bufferCount);
QFETCH(const QAudioFormat::SampleFormat, sampleFormat);
QFETCH(const QAudioFormat::ChannelConfig, channelConfig);
QFETCH(const int, sampleRate);
QFETCH(const milliseconds, duration);
CaptureSessionFixture f{ StreamType::Audio };
QAudioFormat format;
format.setSampleFormat(sampleFormat);
format.setSampleRate(sampleRate);
format.setChannelConfig(channelConfig);
f.m_audioGenerator.setFormat(format);
f.m_audioGenerator.setBufferCount(bufferCount);
f.m_audioGenerator.setDuration(duration);
f.start(RunMode::Pull, AutoStop::EmitEmpty);
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError,
f.m_recorder.errorString().toLatin1().constData());
auto info = MediaInfo::create(f.m_recorder.actualLocation());
QVERIFY(info->m_hasAudio);
QCOMPARE_GE(info->m_duration, duration - 50ms);
QCOMPARE_LE(info->m_duration, duration + 50ms);
QVERIFY(info->m_audioBuffer.isValid());
microseconds audioDataDuration(
info->m_audioBuffer.format().durationForBytes(info->m_audioBuffer.byteCount()));
// TODO: investigate inaccuracies
QCOMPARE_GT(audioDataDuration, duration - 50ms);
QCOMPARE_LT(audioDataDuration, duration + 150ms);
QByteArray sentAudioData = createSineWaveData(format, duration);
const AudioComparisonResult comparisonResult =
compareAudioData(info->m_audioBuffer, QAudioBuffer(sentAudioData, format));
if (format.channelCount() != 1)
QSKIP("Temporary skip checking audio comparison for channels count > 1");
QVERIFY2(comparisonResult.check(), comparisonResult.toString().toLatin1().constData());
}
void tst_QMediaFrameInputsBackend::mediaRecorderWritesVideo_whenVideoFramesInputSendsFrames_data()
{
QTest::addColumn<int>("framesNumber");
QTest::addColumn<milliseconds>("frameDuration");
QTest::addColumn<QSize>("resolution");
QTest::addColumn<bool>("setTimeStamp");
QTest::addRow("framesNumber: 5; frameRate: 2; resolution: 50x80; with time stamps")
<< 5 << 500ms << QSize(50, 80) << true;
QTest::addRow("framesNumber: 20; frameRate: 1; resolution: 200x100; with time stamps")
<< 20 << 1000ms << QSize(200, 100) << true;
QTest::addRow("framesNumber: 20; frameRate: 30; resolution: 200x100; with frame rate")
<< 20 << 250ms << QSize(200, 100) << false;
QTest::addRow("framesNumber: 60; frameRate: 4; resolution: 200x100; with frame rate")
<< 60 << 24ms << QSize(200, 100) << false;
}
void tst_QMediaFrameInputsBackend::mediaRecorderWritesVideo_whenVideoFramesInputSendsFrames()
{
QFETCH(const int, framesNumber);
QFETCH(const milliseconds, frameDuration);
QFETCH(const QSize, resolution);
QFETCH(const bool, setTimeStamp);
CaptureSessionFixture f{ StreamType::Video };
f.m_videoGenerator.setFrameCount(framesNumber);
f.m_videoGenerator.setSize(resolution);
const qreal frameRate = 1e6 / duration_cast<microseconds>(frameDuration).count();
if (setTimeStamp)
f.m_videoGenerator.setPeriod(frameDuration);
else
f.m_videoGenerator.setFrameRate(frameRate);
f.start(RunMode::Pull, AutoStop::EmitEmpty);
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError, f.m_recorder.errorString().toLatin1().constData());
auto info = MediaInfo::create(f.m_recorder.actualLocation());
QCOMPARE_LT(info->m_frameRate, frameRate * 1.001);
QCOMPARE_GT(info->m_frameRate, frameRate * 0.999);
QCOMPARE_LT(info->m_duration, frameDuration * framesNumber * 1.001);
QCOMPARE_GE(info->m_duration, frameDuration * framesNumber * 0.999);
QCOMPARE(info->m_size, resolution);
QCOMPARE_EQ(info->m_frameCount, framesNumber);
}
void tst_QMediaFrameInputsBackend::
sinkReceivesFrameWithTransformParams_whenPresentationTransformPresent_data()
{
QTest::addColumn<QtVideo::Rotation>("presentationRotation");
QTest::addColumn<bool>("presentationMirrored");
QTest::addRow("No rotation, not mirrored") << QtVideo::Rotation::None << false;
QTest::addRow("90 degrees, not mirrored") << QtVideo::Rotation::Clockwise90 << false;
QTest::addRow("180 degrees, not mirrored") << QtVideo::Rotation::Clockwise180 << false;
QTest::addRow("270 degrees, not mirrored") << QtVideo::Rotation::Clockwise270 << false;
QTest::addRow("No rotation, mirrored") << QtVideo::Rotation::None << true;
QTest::addRow("90 degrees, mirrored") << QtVideo::Rotation::Clockwise90 << true;
QTest::addRow("180 degrees, mirrored") << QtVideo::Rotation::Clockwise180 << true;
QTest::addRow("270 degrees, mirrored") << QtVideo::Rotation::Clockwise270 << true;
}
void tst_QMediaFrameInputsBackend::
sinkReceivesFrameWithTransformParams_whenPresentationTransformPresent()
{
QFETCH(const QtVideo::Rotation, presentationRotation);
QFETCH(const bool, presentationMirrored);
// Arrange
CaptureSessionFixture f{ StreamType::Video };
f.m_videoGenerator.setPattern(ImagePattern::ColoredSquares);
f.m_videoGenerator.setFrameCount(2);
f.m_videoGenerator.setPresentationRotation(presentationRotation);
f.m_videoGenerator.setPresentationMirrored(presentationMirrored);
TestVideoSink videoSink{ true /*store frames*/};
f.setVideoSink(&videoSink);
f.start(RunMode::Push, AutoStop::No);
// Act - push two frames
f.m_videoGenerator.nextFrame();
f.m_videoGenerator.nextFrame();
QCOMPARE_EQ(videoSink.m_frameList.size(), 2);
// Assert
const QVideoFrame frame = videoSink.m_frameList.back();
QCOMPARE_EQ(frame.mirrored(), presentationMirrored);
QCOMPARE_EQ(frame.rotation(), presentationRotation);
// Note: Frame data is not transformed and QVideoFrame::toImage does not apply
// transformations. Transformation parameters should be forwarded to rendering
const std::array<QColor, 4> colors = MediaInfo::sampleQuadrants(frame.toImage());
QVERIFY(fuzzyCompare(colors[0], Qt::red));
QVERIFY(fuzzyCompare(colors[1], Qt::green));
QVERIFY(fuzzyCompare(colors[2], Qt::blue));
QVERIFY(fuzzyCompare(colors[3], Qt::yellow));
}
void tst_QMediaFrameInputsBackend::mediaRecorderWritesVideo_withSingleFrame()
{
CaptureSessionFixture f{ StreamType::Video };
f.m_videoGenerator.setFrameCount(1);
f.m_videoGenerator.setSize({ 640, 480 });
f.m_videoGenerator.setPeriod(1s);
f.m_videoGenerator.setFrameRate(1.f);
f.start(RunMode::Pull, AutoStop::EmitEmpty);
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError, f.m_recorder.errorString().toLatin1().constData());
auto info = MediaInfo::create(f.m_recorder.actualLocation());
QCOMPARE_EQ(info->m_frameCount, 1);
QCOMPARE_EQ(info->m_duration, 1s);
}
void tst_QMediaFrameInputsBackend::
mediaRecorderStops_whenEndOfStreamIsSentFromOtherThreadWhileStopping_data()
{
QTest::addColumn<int>("stopDelayMs");
// The race window is while the recorder tears down its encoders after stopping,
// and its timing depends on the machine. So we try a range of delays.
for (int delayMs : { 0, 1, 2, 4, 8, 16 })
QTest::addRow("%dms", delayMs) << delayMs;
}
// With autoStop enabled, an end-of-stream frame sent from a thread other than the
// recorder's own reaches the recorder through a queued connection. This checks that
// stopping the recorder before that queued notification is handled is safe.
void tst_QMediaFrameInputsBackend::
mediaRecorderStops_whenEndOfStreamIsSentFromOtherThreadWhileStopping()
{
QFETCH(const int, stopDelayMs);
CaptureSessionFixture f{ StreamType::Video };
f.m_videoGenerator.setFrameCount(1000);
f.m_recorder.setAutoStop(true);
f.start(RunMode::Push, AutoStop::No);
// Send frames until the recorder is encoding.
QTRY_VERIFY_WITH_TIMEOUT(
[&] {
if (!f.readyToSendVideoFrame.empty())
f.m_videoGenerator.nextFrame();
return f.m_recorder.duration() > 0;
}(),
60s);
// Wait until the frame input accepts another frame.
f.readyToSendVideoFrame.clear();
QTRY_VERIFY(!f.readyToSendVideoFrame.empty());
// Send the end-of-stream frame from another thread. We block until it is sent,
// so that the frame input is never accessed from two threads at the same time.
bool endOfStreamSent = false;
std::thread sender([&] {
endOfStreamSent = f.m_videoInput.sendVideoFrame(QVideoFrame{});
});
sender.join();
QVERIFY(endOfStreamSent);
// Stop before the queued end-of-stream notification is handled. We then block
// the event loop for a while, so the notification is handled while the recorder
// is tearing down its encoders.
f.m_recorder.stop();
std::this_thread::sleep_for(std::chrono::milliseconds{ stopDelayMs });
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError,
f.m_recorder.errorString().toLatin1().constData());
}
void tst_QMediaFrameInputsBackend::
mediaRecorderStops_whenFramesSentFromOtherThread_andEndOfStreamSentFromMainThread_data()
{
QTest::addColumn<int>("senderStartDelayMs");
// Vary the delay before sending EOS to hit different points in the background sender's loop.
for (int delayMs : { 0, 1, 2, 4, 8, 16 })
QTest::addRow("%dms", delayMs) << delayMs;
}
// Checks that sending EOS from the main thread is safe while a background thread
// is actively sending frames — the mirror case of
// mediaRecorderStops_whenEndOfStreamIsSentFromOtherThreadWhileStopping.
void tst_QMediaFrameInputsBackend::
mediaRecorderStops_whenFramesSentFromOtherThread_andEndOfStreamSentFromMainThread()
{
QFETCH(const int, senderStartDelayMs);
CaptureSessionFixture f{ StreamType::Video };
f.m_recorder.setAutoStop(true);
f.start(RunMode::Push, AutoStop::No);
// Send frames from the main thread until the recorder is encoding.
QTRY_VERIFY_WITH_TIMEOUT(
[&] {
if (!f.readyToSendVideoFrame.empty())
f.m_videoGenerator.nextFrame();
return f.m_recorder.duration() > 0;
}(),
60s);
// Wait until the frame input accepts another frame.
f.readyToSendVideoFrame.clear();
QTRY_VERIFY(!f.readyToSendVideoFrame.empty());
// Start a background thread that continuously sends frames.
std::atomic<bool> keepSending{ true };
std::thread sender([&] {
while (keepSending.load(std::memory_order_relaxed))
f.m_videoInput.sendVideoFrame(f.m_videoGenerator.createFrame());
});
std::this_thread::sleep_for(std::chrono::milliseconds{ senderStartDelayMs });
// Send the end-of-stream from the main thread while the background thread
// is still sending frames.
f.m_videoInput.sendVideoFrame(QVideoFrame{});
keepSending.store(false, std::memory_order_relaxed);
sender.join();
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError,
f.m_recorder.errorString().toLatin1().constData());
}
void tst_QMediaFrameInputsBackend::readyToSend_isEmitted_whenRecordingStarts_data()
{
QTest::addColumn<StreamType>("streamType");
QTest::addRow("audio") << StreamType::Audio;
QTest::addRow("video") << StreamType::Video;
QTest::addRow("audioAndVideo") << StreamType::AudioAndVideo;
}
void tst_QMediaFrameInputsBackend::readyToSend_isEmitted_whenRecordingStarts()
{
QFETCH(StreamType, streamType);
CaptureSessionFixture f{ streamType };
f.start(RunMode::Push, AutoStop::No);
if (f.hasAudio())
QTRY_COMPARE_EQ(f.readyToSendAudioBuffer.size(), 1);
if (f.hasVideo())
QTRY_COMPARE_EQ(f.readyToSendVideoFrame.size(), 1);
}
void tst_QMediaFrameInputsBackend::readyToSendVideoFrame_isEmitted_whenSendVideoFrameIsCalled()
{
CaptureSessionFixture f{ StreamType::Video };
f.start(RunMode::Push, AutoStop::No);
QVERIFY(f.readyToSendVideoFrame.wait());
f.m_videoInput.sendVideoFrame(f.m_videoGenerator.createFrame());
QVERIFY(f.readyToSendVideoFrame.wait());
f.m_videoInput.sendVideoFrame(f.m_videoGenerator.createFrame());
QVERIFY(f.readyToSendVideoFrame.wait());
}
void tst_QMediaFrameInputsBackend::readyToSendAudioBuffer_isEmitted_whenSendAudioBufferIsCalled()
{
CaptureSessionFixture f{ StreamType::Audio };
f.start(RunMode::Push, AutoStop::No);
QVERIFY(f.readyToSendAudioBuffer.wait());
f.m_audioInput.sendAudioBuffer(f.m_audioGenerator.createAudioBuffer());
QVERIFY(f.readyToSendAudioBuffer.wait());
f.m_audioInput.sendAudioBuffer(f.m_audioGenerator.createAudioBuffer());
QVERIFY(f.readyToSendAudioBuffer.wait());
}
void tst_QMediaFrameInputsBackend::readyToSendVideoFrame_isEmittedRepeatedly_whenPullModeIsEnabled()
{
CaptureSessionFixture f{ StreamType::Video };
constexpr int expectedSignalCount = 4;
f.m_videoGenerator.setFrameCount(expectedSignalCount - 1);
f.start(RunMode::Pull, AutoStop::EmitEmpty);
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError, f.m_recorder.errorString().toLatin1().constData());
QCOMPARE_EQ(f.readyToSendVideoFrame.size(), expectedSignalCount);
}
void tst_QMediaFrameInputsBackend::
readyToSendAudioBuffer_isEmittedRepeatedly_whenPullModeIsEnabled()
{
CaptureSessionFixture f{ StreamType::Audio };
constexpr int expectedSignalCount = 4;
f.m_audioGenerator.setBufferCount(expectedSignalCount - 1);
f.start(RunMode::Pull, AutoStop::EmitEmpty);
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError, f.m_recorder.errorString().toLatin1().constData());
QCOMPARE_EQ(f.readyToSendAudioBuffer.size(), expectedSignalCount);
}
void tst_QMediaFrameInputsBackend::
readyToSendAudioBufferAndVideoFrame_isEmittedRepeatedly_whenPullModeIsEnabled()
{
CaptureSessionFixture f{ StreamType::AudioAndVideo };
constexpr int expectedSignalCount = 4;
f.m_audioGenerator.setBufferCount(expectedSignalCount - 1);
f.m_videoGenerator.setFrameCount(expectedSignalCount - 1);
f.start(RunMode::Pull, AutoStop::EmitEmpty);
QVERIFY(f.waitForRecorderStopped(60s));
QVERIFY2(f.m_recorder.error() == QMediaRecorder::NoError, f.m_recorder.errorString().toLatin1().constData());
QCOMPARE_EQ(f.readyToSendAudioBuffer.size(), expectedSignalCount);
QCOMPARE_EQ(f.readyToSendVideoFrame.size(), expectedSignalCount);
}
QT_END_NAMESPACE
QTEST_MAIN(tst_QMediaFrameInputsBackend)
#include "moc_tst_qmediaframeinputsbackend.cpp"