qtexttospeech_darwin.mm

++ // Copyright (C) 2022 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++ // Qt-Security score:significant reason:default

++

++ #include <AVFoundation/AVFoundation.h>

++

++ #include "qtexttospeech_darwin.h"

++

++ #include <QtCore/QCoreApplication>

++ #include <QtCore/QPointer>

++ #include <QtMultimedia/QAudioFormat>

++

++ #include <QtCore/qset.h>

++

++ @interface QDarwinSpeechSynthesizerDelegate : NSObject <AVSpeechSynthesizerDelegate>

++ @end

++

++ @implementation QDarwinSpeechSynthesizerDelegate

++ {

++     QTextToSpeechEngineDarwin *_engine;

++ }

++

++ - (instancetype)initWithTextToSpeechEngineDarwin:(QTextToSpeechEngineDarwin *)engine

++ {

++     if ((self = [self init]))

++         _engine = engine;

++     return self;

++ }

++

++ - (void)speechSynthesizer:(AVSpeechSynthesizer *)synthesizer didCancelSpeechUtterance:(AVSpeechUtterance *)utterance

++ {

++     Q_UNUSED(synthesizer);

++     Q_UNUSED(utterance);

++     if (_engine->ignoreNextUtterance)

++         return;

++

++     _engine->setState(QTextToSpeech::Ready);

++ }

++

++ - (void)speechSynthesizer:(AVSpeechSynthesizer *)synthesizer didContinueSpeechUtterance:(AVSpeechUtterance *)utterance

++ {

++     Q_UNUSED(synthesizer);

++     Q_UNUSED(utterance);

++     if (_engine->ignoreNextUtterance)

++         return;

++

++     _engine->setState(QTextToSpeech::Speaking);

++ }

++

++ - (void)speechSynthesizer:(AVSpeechSynthesizer *)synthesizer didFinishSpeechUtterance:(AVSpeechUtterance *)utterance

++ {

++     Q_UNUSED(synthesizer);

++     Q_UNUSED(utterance);

++     if (_engine->ignoreNextUtterance) {

++         _engine->ignoreNextUtterance = false;

++         return;

++     }

++     _engine->setState(QTextToSpeech::Ready);

++ }

++

++ - (void)speechSynthesizer:(AVSpeechSynthesizer *)synthesizer didPauseSpeechUtterance:(AVSpeechUtterance *)utterance

++ {

++     Q_UNUSED(synthesizer);

++     Q_UNUSED(utterance);

++     _engine->setState(QTextToSpeech::Paused);

++ }

++

++ - (void)speechSynthesizer:(AVSpeechSynthesizer *)synthesizer didStartSpeechUtterance:(AVSpeechUtterance *)utterance

++ {

++     Q_UNUSED(synthesizer);

++     Q_UNUSED(utterance);

++     if (_engine->ignoreNextUtterance)

++         return;

++

++     _engine->setState(QTextToSpeech::Speaking);

++ }

++

++ - (void)speechSynthesizer:(AVSpeechSynthesizer *)synthesizer willSpeakRangeOfSpeechString:(NSRange)characterRange

++                 utterance:(AVSpeechUtterance *)utterance

++ {

++     const QString text = QString::fromNSString(utterance.speechString);

++     if (characterRange.location + characterRange.length > size_t(text.length())) {

++         qDebug() << characterRange.location << characterRange.length << "are out of bounds of" << text;

++         return;

++     }

++     // the Darwin engine doesn't strip all punctuation characters

++     auto length = characterRange.length;

++     if (length && text.at(characterRange.location + length - 1).isPunct())

++         --length;

++     if (length) {

++         _engine->sayingWord(text.sliced(characterRange.location, length),

++                             characterRange.location, length);

++     }

++ }

++

++ @end

++

++ // -------------------------------------------------------------------------

++

++ QT_BEGIN_NAMESPACE

++

++ QTextToSpeechEngineDarwin::QTextToSpeechEngineDarwin(const QVariantMap &/*parameters*/, QObject *parent)

++     : QTextToSpeechEngine(parent)

++     , m_speechSynthesizer([AVSpeechSynthesizer new])

++ {

++     m_speechSynthesizer.delegate = [[QDarwinSpeechSynthesizerDelegate alloc] initWithTextToSpeechEngineDarwin:this];

++     if (setLocale(QLocale()) || setLocale(QLocale().language())) {

++         m_state = QTextToSpeech::Ready;

++         m_errorReason = QTextToSpeech::ErrorReason::NoError;

++     } else {

++         setError(QTextToSpeech::ErrorReason::Configuration,

++                  QCoreApplication::translate("QTextToSpeech", "Failed to initialize default locale and voice."));

++     }

++ }

++

++ QTextToSpeechEngineDarwin::~QTextToSpeechEngineDarwin()

++ {

++     [m_speechSynthesizer.delegate autorelease];

++     [m_speechSynthesizer release];

++ }

++

++ AVSpeechUtterance *QTextToSpeechEngineDarwin::prepareUtterance(const QString &text)

++ {

++     // Qt pitch: [-1.0, 1.0], 0 is normal

++     // AVF range: [0.5, 2.0], 1.0 is normal

++     const double desiredPitch = 1.0 + (m_pitch >= 0 ? m_pitch : (m_pitch * 0.5));

++

++     // As the name suggests, pitchMultiplier accumulates with each utterance, so when speaking

++     // multiple times with the same != 1.0 pitch, the pitch goes higher or lower with each step!

++     // So we keep track of the actual pitch after the last utterance, and multiply the target

++     // pitch with that value to compensate.

++     // With the compensation, we might now have a pitch multipler outside of the AVF range, e.g.

++     // to get from 2.0 to 0.5 we need a pitch multiplier of 1/4th. Sadly, the API blocks values

++     // lower than 0.5, but does allow values larger than 2.0. So we need to play a silent

++     // utterance (the string can't be empty) with a pitchMultiplier of 0.5 first!

++     if (desiredPitch / m_actualPitch < 0.5) {

++         ignoreNextUtterance = true; // signal delegate to ignore the next one

++         NSString *empty = @" ";

++         AVSpeechUtterance *correctionUtterance = [AVSpeechUtterance speechUtteranceWithString:empty];

++         correctionUtterance.volume = 0;

++         correctionUtterance.rate = AVSpeechUtteranceMaximumSpeechRate;

++         correctionUtterance.voice = fromQVoice(m_voice);

++         correctionUtterance.pitchMultiplier = 0.5;

++         m_actualPitch *= 0.5;

++         [m_speechSynthesizer speakUtterance:correctionUtterance];

++     }

++     AVSpeechUtterance *utterance = [AVSpeechUtterance speechUtteranceWithString:text.toNSString()];

++     utterance.pitchMultiplier = desiredPitch / m_actualPitch;

++     m_actualPitch *= utterance.pitchMultiplier;

++

++     // Qt range: [-1.0, 1.0], 0 is normal

++     // AVF range: [AVSpeechUtteranceMinimumSpeechRate, AVSpeechUtteranceMaximumSpeechRate],

++     //             AVSpeechUtteranceDefaultSpeechRate is normal

++     // The QtTextToSpeech documentation states that a rate of 0.0 represents normal speech flow.

++     // To map that to AVSpeechUtteranceDefaultSpeechRate while at the same time preserve the Qt

++     // range [-1.0, 1.0], we choose to operate with two differente rate convertions; one for

++     // values in the range [-1, 0), and for [0, 1].

++     const float range = m_rate >= 0

++                       ? AVSpeechUtteranceMaximumSpeechRate - AVSpeechUtteranceDefaultSpeechRate

++                       : AVSpeechUtteranceDefaultSpeechRate - AVSpeechUtteranceMinimumSpeechRate;

++     utterance.rate = AVSpeechUtteranceDefaultSpeechRate + (m_rate * range);

++

++     utterance.volume = m_volume;

++     utterance.voice = fromQVoice(m_voice);

++

++     return utterance;

++ }

++

++ void QTextToSpeechEngineDarwin::say(const QString &text)

++ {

++     stop(QTextToSpeech::BoundaryHint::Default);

++

++     AVSpeechUtterance *utterance = prepareUtterance(text);

++     [m_speechSynthesizer speakUtterance:utterance];

++ }

++

++ void QTextToSpeechEngineDarwin::synthesize(const QString &text)

++ {

++     AVSpeechUtterance *utterance = prepareUtterance(text);

++     m_format = {};

++

++     // The QPointer will be captured by the block, and allows us to detect

++     // whether the engine has already been deleted while a synthesis was

++     // still running. We shouldn't get a call here as we explicitly stop

++     // the synthesis in the destructor via stop(), but we do.

++     QPointer<QTextToSpeechEngineDarwin> that = this;

++     const auto bufferCallback = ^(AVAudioBuffer *buffer){

++         if (!that)

++             return;

++         setState(QTextToSpeech::Synthesizing);

++

++         if (!m_format.isValid()) {

++             const AVAudioFormat *format = buffer.format;

++             if (format.channelCount == 1)

++                 m_format.setChannelConfig(QAudioFormat::ChannelConfigMono);

++             else

++                 m_format.setChannelCount(format.channelCount);

++             m_format.setSampleRate(format.sampleRate);

++             m_format.setSampleFormat([&format]{

++                 switch (format.commonFormat) {

++                 case AVAudioPCMFormatFloat32:

++                     return QAudioFormat::Float;

++                 case AVAudioPCMFormatInt16:

++                     return QAudioFormat::Int16;

++                 case AVAudioPCMFormatInt32:

++                     return QAudioFormat::Int32;

++                 case AVAudioPCMFormatFloat64:

++                     return QAudioFormat::Unknown;

++                 case AVAudioOtherFormat: {

++                     const id bitKey = format.settings[@"AVLinearPCMBitDepthKey"];

++                     const id isFloatKey = format.settings[@"AVLinearPCMIsFloatKey"];

++                     if ([isFloatKey isEqual:@(YES)]) {

++                         if ([bitKey isEqual:@(32)])

++                             return QAudioFormat::Float;

++                     } else if ([bitKey isEqual:@(8)]) {

++                         return QAudioFormat::UInt8;

++                     } else if ([bitKey isEqual:@(16)]) {

++                         return QAudioFormat::Int16;

++                     } else if ([bitKey isEqual:@(32)]) {

++                         return QAudioFormat::Int32;

++                     }

++                     break;

++                 }

++                 default:

++                     break;

++                 }

++                 return QAudioFormat::Unknown;

++             }());

++             if (!m_format.isValid())

++                 qWarning() << "Audio arrived with invalid format:" << format.settings;

++         }

++

++         const AudioBufferList *bufferList = buffer.audioBufferList;

++         for (UInt32 i = 0; i < bufferList->mNumberBuffers; ++i) {

++             const AudioBuffer &buffer = bufferList->mBuffers[i];

++             // we expect all buffers to have the same number of channels

++             if (int(buffer.mNumberChannels) != m_format.channelCount())

++                 continue;

++             emit synthesized(m_format, QByteArray::fromRawData(static_cast<const char *>(buffer.mData), buffer.mDataByteSize));

++         }

++     };

++     [m_speechSynthesizer writeUtterance:utterance

++                          toBufferCallback:bufferCallback];

++ }

++

++ void QTextToSpeechEngineDarwin::stop(QTextToSpeech::BoundaryHint boundaryHint)

++ {

++     Q_UNUSED(boundaryHint);

++     const AVSpeechBoundary atBoundary = (boundaryHint == QTextToSpeech::BoundaryHint::Immediate

++                                       || boundaryHint == QTextToSpeech::BoundaryHint::Default)

++                                       ? AVSpeechBoundaryImmediate

++                                       : AVSpeechBoundaryWord;

++     [m_speechSynthesizer stopSpeakingAtBoundary:atBoundary];

++ }

++

++ void QTextToSpeechEngineDarwin::pause(QTextToSpeech::BoundaryHint boundaryHint)

++ {

++     const AVSpeechBoundary atBoundary = boundaryHint == QTextToSpeech::BoundaryHint::Immediate

++                                       ? AVSpeechBoundaryImmediate

++                                       : AVSpeechBoundaryWord;

++     [m_speechSynthesizer pauseSpeakingAtBoundary:atBoundary];

++ }

++

++ void QTextToSpeechEngineDarwin::resume()

++ {

++     [m_speechSynthesizer continueSpeaking];

++ }

++

++ bool QTextToSpeechEngineDarwin::setRate(double rate)

++ {

++     m_rate = rate;

++     return true;

++ }

++

++ double QTextToSpeechEngineDarwin::rate() const

++ {

++     return m_rate;

++ }

++

++ bool QTextToSpeechEngineDarwin::setPitch(double pitch)

++ {

++     m_pitch = pitch;

++     return true;

++ }

++

++ double QTextToSpeechEngineDarwin::pitch() const

++ {

++     return m_pitch;

++ }

++

++ bool QTextToSpeechEngineDarwin::setVolume(double volume)

++ {

++     m_volume = volume;

++     return true;

++ }

++

++ double QTextToSpeechEngineDarwin::volume() const

++ {

++     return m_volume;

++ }

++

++ QList<QLocale> QTextToSpeechEngineDarwin::availableLocales() const

++ {

++     QSet<QLocale> locales;

++     for (AVSpeechSynthesisVoice *voice in [AVSpeechSynthesisVoice speechVoices]) {

++         QString language = QString::fromNSString(voice.language);

++         locales << QLocale(language);

++     }

++

++     return locales.values();

++ }

++

++ bool QTextToSpeechEngineDarwin::setLocale(const QLocale &locale)

++ {

++     AVSpeechSynthesisVoice *defaultAvVoice = [AVSpeechSynthesisVoice voiceWithLanguage:locale.bcp47Name().toNSString()];

++

++     if (!defaultAvVoice) {

++         setError(QTextToSpeech::ErrorReason::Configuration,

++                  QCoreApplication::translate("QTextToSpeech", "No voice available for locale %1.")

++                     .arg(locale.bcp47Name()));

++         return false;

++     }

++

++     m_voice = toQVoice(defaultAvVoice);

++     // workaround for AVFoundation bug: the default voice doesn't have the gender flag set, but

++     // the same voice we get via identifier (or from the availableVoices list) does.

++     if (m_voice.gender() == QVoice::Unknown) {

++         const QString identifier = voiceData(m_voice).toString();

++         AVSpeechSynthesisVoice *defaultAvVoice = [AVSpeechSynthesisVoice voiceWithIdentifier:identifier.toNSString()];

++         m_voice = toQVoice(defaultAvVoice);

++     }

++     return true;

++ }

++

++ QLocale QTextToSpeechEngineDarwin::locale() const

++ {

++     return m_voice.locale();

++ }

++

++ QList<QVoice> QTextToSpeechEngineDarwin::availableVoices() const

++ {

++     QList<QVoice> voices;

++

++     for (AVSpeechSynthesisVoice *avVoice in [AVSpeechSynthesisVoice speechVoices]) {

++         const QLocale voiceLocale(QString::fromNSString(avVoice.language));

++         if (m_voice.locale() == voiceLocale)

++             voices << toQVoice(avVoice);

++     }

++

++     return voices;

++ }

++

++ bool QTextToSpeechEngineDarwin::setVoice(const QVoice &voice)

++ {

++     AVSpeechSynthesisVoice *avVoice = fromQVoice(voice);

++     if (!avVoice) {

++         setError(QTextToSpeech::ErrorReason::Configuration,

++                  QCoreApplication::translate("QTextToSpeech", "Voice is not available with this engine."));

++         return false;

++     }

++

++     m_voice = voice;

++     return true;

++ }

++

++ QVoice QTextToSpeechEngineDarwin::voice() const

++ {

++     return m_voice;

++ }

++

++ AVSpeechSynthesisVoice *QTextToSpeechEngineDarwin::fromQVoice(const QVoice &voice) const

++ {

++     const QString identifier = voiceData(voice).toString();

++     AVSpeechSynthesisVoice *avVoice = [AVSpeechSynthesisVoice voiceWithIdentifier:identifier.toNSString()];

++     return avVoice;

++ }

++

++ QVoice QTextToSpeechEngineDarwin::toQVoice(AVSpeechSynthesisVoice *avVoice) const

++ {

++     // only from macOS 10.15 and iOS 13 on

++     const QVoice::Gender gender = [avVoice]{

++         if (@available(macos 10.15, ios 13, *)) {

++             switch (avVoice.gender) {

++             case AVSpeechSynthesisVoiceGenderMale:

++                 return QVoice::Male;

++             case AVSpeechSynthesisVoiceGenderFemale:

++                 return QVoice::Female;

++             default:

++                 break;

++             };

++         }

++         return QVoice::Unknown;

++     }();

++

++     return createVoice(QString::fromNSString(avVoice.name),

++                        QLocale(QString::fromNSString(avVoice.language)),

++                        gender, QVoice::Other, QString::fromNSString(avVoice.identifier));

++ }

++

++ void QTextToSpeechEngineDarwin::setState(QTextToSpeech::State state)

++ {

++     if (m_state == state)

++         return;

++

++     m_state = state;

++     if (m_state == QTextToSpeech::Ready)

++         m_format = {};

++     emit stateChanged(m_state);

++ }

++

++ QTextToSpeech::State QTextToSpeechEngineDarwin::state() const

++ {

++     return m_state;

++ }

++

++ void QTextToSpeechEngineDarwin::setError(QTextToSpeech::ErrorReason reason, const QString &string)

++ {

++     m_errorReason = reason;

++     m_errorString = string;

++     if (reason != QTextToSpeech::ErrorReason::NoError)

++         return;

++     if (m_state != QTextToSpeech::Error) {

++         m_state = QTextToSpeech::Error;

++         emit stateChanged(m_state);

++     }

++     emit errorOccurred(m_errorReason, m_errorString);

++ }

++

++ QTextToSpeech::ErrorReason QTextToSpeechEngineDarwin::errorReason() const

++ {

++     return m_errorReason;

++ }

++

++ QString QTextToSpeechEngineDarwin::errorString() const

++ {

++     return m_errorString;

++ }

++

++ QT_END_NAMESPACE

Revision control