Revision control
++ // Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies).
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++
++
++ #include "avfmediaencoder_p.h"
++
++ #include <camera/avfcamera_p.h>
++ #include <camera/avfcamerarenderer_p.h>
++ #include <camera/avfcameraservice_p.h>
++ #include <camera/avfcamerasession_p.h>
++ #include <qdarwinformatsinfo_p.h>
++
++ #include <QtMultimedia/qaudiodevice.h>
++ #include <QtMultimedia/qmediadevices.h>
++ #include <QtMultimedia/private/qavfcameradebug_p.h>
++ #include <QtMultimedia/private/qavfcamerautility_p.h>
++ #include <QtMultimedia/private/qcoreaudioutils_p.h>
++ #include <QtMultimedia/private/qmediarecorder_p.h>
++ #include <QtMultimedia/private/qmediastoragelocation_p.h>
++ #include <QtMultimedia/private/qplatformaudioinput_p.h>
++ #include <QtMultimedia/private/qplatformaudiooutput_p.h>
++ #include <QtCore/qdebug.h>
++ #include <QtCore/qmath.h>
++
++ QT_USE_NAMESPACE
++
++ namespace {
++
++ bool qt_is_writable_file_URL(NSURL *fileURL)
++ {
++ Q_ASSERT(fileURL);
++
++ if (![fileURL isFileURL])
++ return false;
++
++ if (NSString *path = [[fileURL path] stringByExpandingTildeInPath]) {
++ return [[NSFileManager defaultManager]
++ isWritableFileAtPath:[path stringByDeletingLastPathComponent]];
++ }
++
++ return false;
++ }
++
++ bool qt_file_exists(NSURL *fileURL)
++ {
++ Q_ASSERT(fileURL);
++
++ if (NSString *path = [[fileURL path] stringByExpandingTildeInPath])
++ return [[NSFileManager defaultManager] fileExistsAtPath:path];
++
++ return false;
++ }
++
++ } // namespace
++
++ AVFMediaEncoder::AVFMediaEncoder(QMediaRecorder *parent)
++ : QObject(parent)
++ , QPlatformMediaRecorder(parent)
++ , m_state(QMediaRecorder::StoppedState)
++ , m_duration(0)
++ , m_audioSettings(nil)
++ , m_videoSettings(nil)
++ //, m_restoreFPS(-1, -1)
++ {
++ m_writer.reset([[QT_MANGLE_NAMESPACE(AVFMediaAssetWriter) alloc] initWithDelegate:this]);
++ if (!m_writer) {
++ qCDebug(qLcCamera) << Q_FUNC_INFO << "failed to create an asset writer";
++ return;
++ }
++ }
++
++ AVFMediaEncoder::~AVFMediaEncoder()
++ {
++ [m_writer abort];
++
++ if (m_audioSettings)
++ [m_audioSettings release];
++ if (m_videoSettings)
++ [m_videoSettings release];
++ }
++
++ bool AVFMediaEncoder::isLocationWritable(const QUrl &location) const
++ {
++ return location.scheme() == QLatin1String("file") || location.scheme().isEmpty();
++ }
++
++ QMediaRecorder::RecorderState AVFMediaEncoder::state() const
++ {
++ return m_state;
++ }
++
++ qint64 AVFMediaEncoder::duration() const
++ {
++ return m_duration;
++ }
++
++ void AVFMediaEncoder::updateDuration(qint64 duration)
++ {
++ m_duration = duration;
++ durationChanged(m_duration);
++ }
++
++ static NSDictionary *avfAudioSettings(const QMediaEncoderSettings &encoderSettings, const QAudioFormat &format)
++ {
++ NSMutableDictionary *settings = [NSMutableDictionary dictionary];
++
++ // Codec
++ int codecId = QDarwinFormatInfo::audioFormatForCodec(encoderSettings.mediaFormat().audioCodec());
++ settings[AVFormatIDKey] = @(codecId);
++
++ // Setting AVEncoderQualityKey is not allowed when format ID is alac or lpcm
++ if (codecId != kAudioFormatAppleLossless && codecId != kAudioFormatLinearPCM
++ && encoderSettings.encodingMode() == QMediaRecorder::ConstantQualityEncoding) {
++ // AudioQuality
++ int quality;
++ switch (encoderSettings.quality()) {
++ case QMediaRecorder::VeryLowQuality:
++ quality = AVAudioQualityMin;
++ break;
++ case QMediaRecorder::LowQuality:
++ quality = AVAudioQualityLow;
++ break;
++ case QMediaRecorder::HighQuality:
++ quality = AVAudioQualityHigh;
++ break;
++ case QMediaRecorder::VeryHighQuality:
++ quality = AVAudioQualityMax;
++ break;
++ case QMediaRecorder::NormalQuality:
++ default:
++ quality = AVAudioQualityMedium;
++ break;
++ }
++ settings[AVEncoderAudioQualityKey] = @(quality);
++ } else {
++ // BitRate
++ bool isBitRateSupported = false;
++ int bitRate = encoderSettings.audioBitRate();
++ if (bitRate > 0) {
++ QList<AudioValueRange> bitRates = qt_supported_bit_rates_for_format(codecId);
++ for (int i = 0; i < bitRates.count(); i++) {
++ if (bitRate >= bitRates[i].mMinimum &&
++ bitRate <= bitRates[i].mMaximum) {
++ isBitRateSupported = true;
++ break;
++ }
++ }
++ if (isBitRateSupported)
++ settings[AVEncoderBitRateKey] = @(encoderSettings.audioBitRate());
++ }
++ }
++
++ // SampleRate
++ int sampleRate = encoderSettings.audioSampleRate();
++ bool isSampleRateSupported = false;
++ if (sampleRate >= 8000 && sampleRate <= 192000) {
++ QList<AudioValueRange> sampleRates = qt_supported_sample_rates_for_format(codecId);
++ for (int i = 0; i < sampleRates.count(); i++) {
++ if (sampleRate >= sampleRates[i].mMinimum && sampleRate <= sampleRates[i].mMaximum) {
++ isSampleRateSupported = true;
++ break;
++ }
++ }
++ }
++ if (!isSampleRateSupported)
++ sampleRate = 44100;
++ settings[AVSampleRateKey] = @(sampleRate);
++
++ // Channels
++ int channelCount = encoderSettings.audioChannelCount();
++ bool isChannelCountSupported = false;
++ if (channelCount > 0) {
++ std::optional<QList<UInt32>> channelCounts = qt_supported_channel_counts_for_format(codecId);
++ // An std::nullopt result indicates that
++ // any number of channels can be encoded.
++ if (channelCounts == std::nullopt) {
++ isChannelCountSupported = true;
++ } else {
++ for (int i = 0; i < channelCounts.value().count(); i++) {
++ if ((UInt32)channelCount == channelCounts.value()[i]) {
++ isChannelCountSupported = true;
++ break;
++ }
++ }
++ }
++
++ // if channel count is provided and it's bigger than 2
++ // provide a supported channel layout
++ if (isChannelCountSupported && channelCount > 2) {
++ AudioChannelLayout channelLayout;
++ memset(&channelLayout, 0, sizeof(AudioChannelLayout));
++ auto channelLayoutTags = qt_supported_channel_layout_tags_for_format(codecId, channelCount);
++ if (channelLayoutTags.size()) {
++ channelLayout.mChannelLayoutTag = channelLayoutTags.first();
++ settings[AVChannelLayoutKey] = [NSData dataWithBytes:&channelLayout length:sizeof(channelLayout)];
++ } else {
++ isChannelCountSupported = false;
++ }
++ }
++
++ if (isChannelCountSupported)
++ settings[AVNumberOfChannelsKey] = @(channelCount);
++ }
++
++ if (!isChannelCountSupported) {
++ // fallback to providing channel layout if channel count is not specified or supported
++ UInt32 size = 0;
++ if (format.isValid()) {
++ auto layout = QCoreAudioUtils::toAudioChannelLayout(format, &size);
++ UInt32 layoutSize = offsetof(AudioChannelLayout, mChannelDescriptions)
++ + layout->mNumberChannelDescriptions * sizeof(AudioChannelDescription);
++ settings[AVChannelLayoutKey] = [NSData dataWithBytes:layout.get() length:layoutSize];
++ } else {
++ // finally default to setting channel count to 1
++ settings[AVNumberOfChannelsKey] = @(1);
++ }
++ }
++
++ if (codecId == kAudioFormatAppleLossless)
++ settings[AVEncoderBitDepthHintKey] = @(24);
++
++ if (codecId == kAudioFormatLinearPCM) {
++ settings[AVLinearPCMBitDepthKey] = @(16);
++ settings[AVLinearPCMIsBigEndianKey] = @NO;
++ settings[AVLinearPCMIsFloatKey] = @NO;
++ settings[AVLinearPCMIsNonInterleaved] = @NO;
++ }
++
++ return settings;
++ }
++
++ static NSDictionary *avfVideoSettings(QMediaEncoderSettings &encoderSettings,
++ AVCaptureDevice *device, AVCaptureConnection *connection,
++ QSize nativeSize)
++ {
++ if (!device)
++ return nil;
++
++
++ // ### re-add needFpsChange
++ // AVFPSRange currentFps = qt_current_framerates(device, connection);
++
++ NSMutableDictionary *videoSettings = [NSMutableDictionary dictionary];
++
++ // -- Codec
++
++ // AVVideoCodecKey is the only mandatory key
++ auto codec = encoderSettings.mediaFormat().videoCodec();
++ NSString *c = QDarwinFormatInfo::videoFormatForCodec(codec);
++ [videoSettings setObject:c forKey:AVVideoCodecKey];
++ [c release];
++
++ // -- Resolution
++
++ int w = encoderSettings.videoResolution().width();
++ int h = encoderSettings.videoResolution().height();
++
++ if (AVCaptureDeviceFormat *currentFormat = device.activeFormat) {
++ CMFormatDescriptionRef formatDesc = currentFormat.formatDescription;
++ CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDesc);
++ FourCharCode formatCodec = CMVideoFormatDescriptionGetCodecType(formatDesc);
++
++ // We have to change the device's activeFormat in 3 cases:
++ // - the requested recording resolution is higher than the current device resolution
++ // - the requested recording resolution has a different aspect ratio than the current device aspect ratio
++ // - the requested frame rate is not available for the current device format
++ AVCaptureDeviceFormat *newFormat = nil;
++ if ((w <= 0 || h <= 0)
++ && encoderSettings.videoFrameRate() > 0
++ && !qt_format_supports_framerate(currentFormat, encoderSettings.videoFrameRate())) {
++
++ newFormat = qt_find_best_framerate_match(device,
++ formatCodec,
++ encoderSettings.videoFrameRate());
++
++ } else if (w > 0 && h > 0) {
++ AVCaptureDeviceFormat *f = qt_find_best_resolution_match(device,
++ encoderSettings.videoResolution(),
++ formatCodec);
++
++ if (f) {
++ CMVideoDimensions d = CMVideoFormatDescriptionGetDimensions(f.formatDescription);
++ qreal fAspectRatio = qreal(d.width) / d.height;
++
++ if (w > dim.width || h > dim.height
++ || qAbs((qreal(dim.width) / dim.height) - fAspectRatio) > 0.01) {
++ newFormat = f;
++ }
++ }
++ }
++
++ if (qt_set_active_format(device, newFormat, false /*### !needFpsChange*/)) {
++ formatDesc = newFormat.formatDescription;
++ dim = CMVideoFormatDescriptionGetDimensions(formatDesc);
++ }
++
++ if (w < 0 || h < 0) {
++ w = dim.width;
++ h = dim.height;
++ }
++
++
++ if (w > 0 && h > 0) {
++ // Make sure the recording resolution has the same aspect ratio as the device's
++ // current resolution
++ qreal deviceAspectRatio = qreal(dim.width) / dim.height;
++ qreal recAspectRatio = qreal(w) / h;
++ if (qAbs(deviceAspectRatio - recAspectRatio) > 0.01) {
++ if (recAspectRatio > deviceAspectRatio)
++ w = qRound(h * deviceAspectRatio);
++ else
++ h = qRound(w / deviceAspectRatio);
++ }
++
++ // recording resolution can't be higher than the device's active resolution
++ w = qMin(w, dim.width);
++ h = qMin(h, dim.height);
++ }
++ }
++
++ if (w > 0 && h > 0) {
++ // Width and height must be divisible by 2
++ w += w & 1;
++ h += h & 1;
++
++ bool isPortrait = nativeSize.width() < nativeSize.height();
++ // Make sure the video has the right aspect ratio
++ if (isPortrait && h < w)
++ qSwap(w, h);
++ else if (!isPortrait && w < h)
++ qSwap(w, h);
++
++ encoderSettings.setVideoResolution(QSize(w, h));
++ } else {
++ w = nativeSize.width();
++ h = nativeSize.height();
++ encoderSettings.setVideoResolution(nativeSize);
++ }
++ videoSettings[AVVideoWidthKey] = @(w);
++ videoSettings[AVVideoHeightKey] = @(h);
++
++ // -- FPS
++
++ if (true /*needFpsChange*/) {
++ const qreal fps = encoderSettings.videoFrameRate();
++ qt_set_framerate_limits(device, connection, fps, fps);
++ }
++ encoderSettings.setVideoFrameRate(qt_current_framerates(device, connection).second);
++
++ // -- Codec Settings
++
++ NSMutableDictionary *codecProperties = [NSMutableDictionary dictionary];
++ int bitrate = -1;
++ float quality = -1.f;
++
++ if (encoderSettings.encodingMode() == QMediaRecorder::ConstantQualityEncoding) {
++ if (encoderSettings.quality() != QMediaRecorder::NormalQuality) {
++ if (codec != QMediaFormat::VideoCodec::MotionJPEG) {
++ qWarning("ConstantQualityEncoding is not supported for MotionJPEG");
++ } else {
++ switch (encoderSettings.quality()) {
++ case QMediaRecorder::VeryLowQuality:
++ quality = 0.f;
++ break;
++ case QMediaRecorder::LowQuality:
++ quality = 0.25f;
++ break;
++ case QMediaRecorder::HighQuality:
++ quality = 0.75f;
++ break;
++ case QMediaRecorder::VeryHighQuality:
++ quality = 1.f;
++ break;
++ default:
++ quality = -1.f; // NormalQuality, let the system decide
++ break;
++ }
++ }
++ }
++ } else if (encoderSettings.encodingMode() == QMediaRecorder::AverageBitRateEncoding){
++ if (codec != QMediaFormat::VideoCodec::H264 && codec != QMediaFormat::VideoCodec::H265)
++ qWarning() << "AverageBitRateEncoding is not supported for codec" << QMediaFormat::videoCodecName(codec);
++ else
++ bitrate = encoderSettings.videoBitRate();
++ } else {
++ qWarning("Encoding mode is not supported");
++ }
++
++ if (bitrate != -1)
++ codecProperties[AVVideoAverageBitRateKey] = @(bitrate);
++ if (quality != -1.f)
++ codecProperties[AVVideoQualityKey] = @(quality);
++
++ videoSettings[AVVideoCompressionPropertiesKey] = codecProperties;
++
++ return videoSettings;
++ }
++
++ void AVFMediaEncoder::applySettings(QMediaEncoderSettings &settings)
++ {
++ unapplySettings();
++
++ AVFCameraSession *session = m_service->session();
++
++ // audio settings
++ const auto audioInput = m_service->audioInput();
++ const QAudioFormat audioFormat = audioInput ? audioInput->device.preferredFormat() : QAudioFormat();
++ m_audioSettings = avfAudioSettings(settings, audioFormat);
++ if (m_audioSettings)
++ [m_audioSettings retain];
++
++ // video settings
++ AVCaptureDevice *device = session->videoCaptureDevice();
++ if (!device)
++ return;
++ const AVFConfigurationLock lock(device); // prevents activeFormat from being overridden
++ AVCaptureConnection *conn = [session->videoOutput()->videoDataOutput() connectionWithMediaType:AVMediaTypeVideo];
++ auto nativeSize = session->videoOutput()->nativeSize();
++ m_videoSettings = avfVideoSettings(settings, device, conn, nativeSize);
++ if (m_videoSettings)
++ [m_videoSettings retain];
++ }
++
++ void AVFMediaEncoder::unapplySettings()
++ {
++ if (m_audioSettings) {
++ [m_audioSettings release];
++ m_audioSettings = nil;
++ }
++ if (m_videoSettings) {
++ [m_videoSettings release];
++ m_videoSettings = nil;
++ }
++ }
++
++ void AVFMediaEncoder::setMetaData(const QMediaMetaData &metaData)
++ {
++ m_metaData = metaData;
++ }
++
++ QMediaMetaData AVFMediaEncoder::metaData() const
++ {
++ return m_metaData;
++ }
++
++ void AVFMediaEncoder::setCaptureSession(QPlatformMediaCaptureSession *session)
++ {
++ AVFCameraService *captureSession = static_cast<AVFCameraService *>(session);
++ if (m_service == captureSession)
++ return;
++
++ if (m_service)
++ stop();
++
++ m_service = captureSession;
++ if (!m_service)
++ return;
++
++ connect(m_service, &AVFCameraService::cameraChanged, this, &AVFMediaEncoder::onCameraChanged);
++ onCameraChanged();
++ }
++
++ void AVFMediaEncoder::record(QMediaEncoderSettings &settings)
++ {
++ if (!m_service || !m_service->session()) {
++ qWarning() << Q_FUNC_INFO << "Encoder is not set to a capture session";
++ return;
++ }
++
++ if (!m_writer) {
++ qCDebug(qLcCamera) << Q_FUNC_INFO << "Invalid recorder";
++ return;
++ }
++
++ if (QMediaRecorder::RecordingState == m_state)
++ return;
++
++ AVFCamera *cameraControl = m_service->avfCameraControl();
++ auto audioInput = m_service->audioInput();
++
++ if (!cameraControl && !audioInput) {
++ qWarning() << Q_FUNC_INFO << "Cannot record without any inputs";
++ updateError(QMediaRecorder::ResourceError, tr("No inputs specified"));
++ return;
++ }
++
++ // This is necessary to explicitly recreate m_audioInput inside AVFCameraSession.
++ // Which in turn is necessary for the case when the microphone was disconnected
++ // after stopping recording and reconnected.
++ m_service->session()->updateAudioInput();
++
++ m_service->session()->setActive(true);
++ const bool audioOnly = settings.videoCodec() == QMediaFormat::VideoCodec::Unspecified;
++ AVCaptureSession *session = m_service->session()->captureSession();
++ float rotation = 0;
++
++ if (!audioOnly) {
++ if (!cameraControl || !cameraControl->isActive()) {
++ qCDebug(qLcCamera) << Q_FUNC_INFO << "can not start record while camera is not active";
++ updateError(QMediaRecorder::ResourceError,
++ QMediaRecorderPrivate::msgFailedStartRecording());
++ return;
++ }
++ }
++
++ const QString path(outputLocation().scheme() == QLatin1String("file") ?
++ outputLocation().path() : outputLocation().toString());
++ const QUrl fileURL(QUrl::fromLocalFile(QMediaStorageLocation::generateFileName(
++ path, audioOnly ? QStandardPaths::MusicLocation : QStandardPaths::MoviesLocation,
++ settings.preferredSuffix())));
++
++ NSURL *nsFileURL = fileURL.toNSURL();
++ if (!nsFileURL) {
++ qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL;
++ updateError(QMediaRecorder::ResourceError, tr("Invalid output file URL"));
++ return;
++ }
++ if (!qt_is_writable_file_URL(nsFileURL)) {
++ qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL
++ << "(the location is not writable)";
++ updateError(QMediaRecorder::ResourceError, tr("Non-writeable file location"));
++ return;
++ }
++ if (qt_file_exists(nsFileURL)) {
++ // We test for/handle this error here since AWAssetWriter will raise an
++ // Objective-C exception, which is not good at all.
++ qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL
++ << "(file already exists)";
++ updateError(QMediaRecorder::ResourceError, tr("File already exists"));
++ return;
++ }
++
++ applySettings(settings);
++
++ QVideoOutputOrientationHandler::setIsRecording(true);
++
++ // We stop session now so that no more frames for renderer's queue
++ // generated, will restart in assetWriterStarted.
++ [session stopRunning];
++
++ if ([m_writer setupWithFileURL:nsFileURL
++ cameraService:m_service
++ audioSettings:m_audioSettings
++ videoSettings:m_videoSettings
++ fileFormat:settings.fileFormat()
++ transform:CGAffineTransformMakeRotation(qDegreesToRadians(rotation))]) {
++
++ m_state = QMediaRecorder::RecordingState;
++
++ actualLocationChanged(fileURL);
++ stateChanged(m_state);
++
++ // Apple recommends to call startRunning and do all
++ // setup on a special queue, and that's what we had
++ // initially (dispatch_async to writerQueue). Unfortunately,
++ // writer's queue is not the only queue/thread that can
++ // access/modify the session, and as a result we have
++ // all possible data/race-conditions with Obj-C exceptions
++ // at best and something worse in general.
++ // Now we try to only modify session on the same thread.
++ [m_writer start];
++ } else {
++ [session startRunning];
++ updateError(QMediaRecorder::FormatError, QMediaRecorderPrivate::msgFailedStartRecording());
++ }
++ }
++
++ void AVFMediaEncoder::pause()
++ {
++ if (!m_service || !m_service->session() || state() != QMediaRecorder::RecordingState)
++ return;
++
++ toggleRecord(false);
++ m_state = QMediaRecorder::PausedState;
++ stateChanged(m_state);
++ }
++
++ void AVFMediaEncoder::resume()
++ {
++ if (!m_service || !m_service->session() || state() != QMediaRecorder::PausedState)
++ return;
++
++ toggleRecord(true);
++ m_state = QMediaRecorder::RecordingState;
++ stateChanged(m_state);
++ }
++
++ void AVFMediaEncoder::stop()
++ {
++ if (m_state != QMediaRecorder::StoppedState) {
++ // Do not check the camera status, we can stop if we started.
++ stopWriter();
++ }
++ QVideoOutputOrientationHandler::setIsRecording(false);
++ }
++
++
++ void AVFMediaEncoder::toggleRecord(bool enable)
++ {
++ if (!m_service || !m_service->session())
++ return;
++
++ if (!enable)
++ [m_writer pause];
++ else
++ [m_writer resume];
++ }
++
++ void AVFMediaEncoder::assetWriterStarted()
++ {
++ }
++
++ void AVFMediaEncoder::assetWriterFinished()
++ {
++
++ const QMediaRecorder::RecorderState lastState = m_state;
++
++ unapplySettings();
++
++ if (m_service) {
++ AVFCameraSession *session = m_service->session();
++
++ if (session->videoOutput()) {
++ session->videoOutput()->resetCaptureDelegate();
++ }
++ if (session->audioPreviewDelegate()) {
++ [session->audioPreviewDelegate() resetAudioPreviewDelegate];
++ }
++ if (session->videoOutput() || session->audioPreviewDelegate())
++ [session->captureSession() startRunning];
++ }
++
++ m_state = QMediaRecorder::StoppedState;
++ if (m_state != lastState)
++ stateChanged(m_state);
++ }
++
++ void AVFMediaEncoder::assetWriterError(QString err)
++ {
++ updateError(QMediaRecorder::FormatError, err);
++ if (m_state != QMediaRecorder::StoppedState)
++ stopWriter();
++ }
++
++ void AVFMediaEncoder::onCameraChanged()
++ {
++ if (m_service && m_service->avfCameraControl()) {
++ AVFCamera *cameraControl = m_service->avfCameraControl();
++ connect(cameraControl, SIGNAL(activeChanged(bool)),
++ SLOT(cameraActiveChanged(bool)));
++ }
++ }
++
++ void AVFMediaEncoder::cameraActiveChanged(bool active)
++ {
++ Q_ASSERT(m_service);
++ AVFCamera *cameraControl = m_service->avfCameraControl();
++ Q_ASSERT(cameraControl);
++
++ if (!active) {
++ return stopWriter();
++ }
++ }
++
++ void AVFMediaEncoder::stopWriter()
++ {
++ [m_writer stop];
++ }
++
++ #include "moc_avfmediaencoder_p.cpp"