qtexttospeech_winrt_audiosource.cpp

++ // Copyright (C) 2022 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++ // Qt-Security score:critical reason:data-parser

++

++ #include "qtexttospeech_winrt_audiosource.h"

++

++ #include <QtCore/qdebug.h>

++ #include <QtCore/qspan.h>

++ #include <QtCore/qtimer.h>

++

++ #include <QtCore/private/qfunctions_winrt_p.h>

++ #include <QtCore/private/qsystemerror_p.h>

++

++ #include <robuffer.h>

++ #include <winrt/base.h>

++ #include <QtCore/private/qfactorycacheregistration_p.h>

++ #include <windows.foundation.h>

++ #include <windows.foundation.collections.h>

++ #include <windows.media.core.h>

++ #include <windows.media.speechsynthesis.h>

++ #include <windows.storage.streams.h>

++

++ #include <wrl.h>

++

++ using namespace ABI::Windows::Foundation;

++ using namespace ABI::Windows::Foundation::Collections;

++ using namespace ABI::Windows::Media::Core;

++ using namespace ABI::Windows::Media::SpeechSynthesis;

++ using namespace ABI::Windows::Storage::Streams;

++ using namespace Microsoft::WRL;

++ using namespace Microsoft::WRL::Wrappers;

++

++ QT_BEGIN_NAMESPACE

++

++ /*

++     AudioSource implements a sequential QIODevice for a stream of synthesized speech.

++     It also implements the handler interfaces for responding to the asynchronous

++     operations of the synthesized speech stream being avialable, and reading data from

++     the stream via a COM buffer.

++

++     Whenever the QIODevice has read all the data from the COM buffer, more data is

++     requested. When the data is available (the BytesReadyHandler's handler's Invoke

++     implementation is called), readyRead is emitted.

++

++     The AudioSource directly controls a QAudioSink. As soon as the data stream is

++     available, the source calls QAudioSink::start. Pause/resume are delegated to the

++     sink; closing the source stops the sink.

++ */

++ AudioSource::AudioSource(ComPtr<IAsyncOperation<SpeechSynthesisStream*>> synthOperation)

++     : synthOperation(synthOperation)

++ {

++     synthOperation->put_Completed(this);

++

++     audioFormat.setSampleFormat(QAudioFormat::Int16);

++     audioFormat.setSampleRate(16000);

++     audioFormat.setChannelConfig(QAudioFormat::ChannelConfigMono);

++ }

++

++ /*

++     Calls close, which is virtual and called from ~QIODevice,

++     but that won't call our override.

++ */

++ AudioSource::~AudioSource()

++ {

++     close();

++ }

++

++ /*

++     Cancel any incomplete asynchronous operations, and stop the

++     sink before closing the QIODevice.

++ */

++ void AudioSource::close()

++ {

++     ComPtr<IAsyncInfo> asyncInfo;

++     AsyncStatus status = AsyncStatus::Completed;

++     if (synthOperation) {

++         if (HRESULT hr = synthOperation.As(&asyncInfo); SUCCEEDED(hr)) {

++             asyncInfo->get_Status(&status);

++             if (status != AsyncStatus::Completed)

++                 asyncInfo->Cancel();

++         }

++     }

++     if (readOperation) {

++         if (HRESULT hr = readOperation.As(&asyncInfo); SUCCEEDED(hr)) {

++             asyncInfo->get_Status(&status);

++             if (status != AsyncStatus::Completed)

++                 asyncInfo->Cancel();

++         }

++     }

++     QIODevice::close();

++ }

++

++ qint64 AudioSource::bytesAvailable() const

++ {

++     return m_bufferSpan.size_bytes() + QIODevice::bytesAvailable();

++ }

++

++ /*

++     Fills data with as many bytes from the COM buffer as possible. If this

++     empties the COM buffer, calls fetchMore to start a new asynchronous

++     read operation.

++ */

++ qint64 AudioSource::readData(char *data, qint64 maxlen)

++ {

++     // this may happen as per the documentation

++     if (!maxlen)

++         return 0;

++

++     if (m_bufferSpan.isEmpty() && atEnd())

++         return -1;

++

++     auto take = [](auto span, qsizetype n) {

++         return span.first(std::min(span.size(), n));

++     };

++

++     auto drop = [](auto span, qsizetype n) {

++         return span.subspan(std::min(span.size(), n));

++     };

++

++     QSpan source = take(m_bufferSpan, maxlen);

++

++     switch (m_pause) {

++     case NoPause:

++         break;

++     case PauseRequested: {

++         Q_ASSERT(audioFormat.sampleFormat() == QAudioFormat::Int16);

++

++         if (m_pauseRequestedAt) {

++             if (m_pauseRequestedAt <= m_bytesRead) {

++                 // we missed the window, pause immediately

++                 source = {};

++             } else if (m_pauseRequestedAt <= m_bytesRead + quint64(source.size())) {

++                 source = take(source, qsizetype(m_pauseRequestedAt - m_bytesRead));

++             } else {

++                 break; // pause point is in a later chunk

++             }

++             m_pause = Paused;

++             m_pauseRequestedAt = 0;

++             break;

++         }

++         // If no byte to pause at is specified, look for silence in the current

++         // chunk. We are dealing with artificially created sound, so we don't have

++         // to find a large enough window with overall low energy; we can just

++         // look for a series (e.g. 1/50th of a second) of samples with low

++         // absolute values.

++         const int silenceDuration = audioFormat.sampleRate() / 50;

++         const auto samples = QSpan(reinterpret_cast<const short *>(source.data()),

++                                    source.size() / sizeof(short));

++

++         if (!m_pauseDetectionSilenceCount.has_value())

++             m_pauseDetectionSilenceCount = 0;

++

++         for (qsizetype i = 0; i < samples.size(); ++i) {

++             if (qAbs(samples[i]) < 10)

++                 *m_pauseDetectionSilenceCount += 1;

++             else {

++                 *m_pauseDetectionSilenceCount = 0;

++                 continue;

++             }

++             if (*m_pauseDetectionSilenceCount > silenceDuration) {

++                 // long enough silence found, only provide data until we are in the silence.

++                 // we will still try to play at least half of the silent part

++                 const qsizetype silentSamplesToPlay = silenceDuration / 2;

++                 source = take(source,

++                               (i > silentSamplesToPlay ? i - silentSamplesToPlay : i)

++                                       * sizeof(short));

++                 m_pause = Paused;

++                 m_pauseDetectionSilenceCount = std::nullopt;

++                 break;

++             } else {

++                 *m_pauseDetectionSilenceCount = 0;

++             }

++         }

++         // no silence found - stop after this chunk

++         if (m_pause != Paused) {

++             m_pauseDetectionSilenceCount = std::nullopt;

++             m_pause = Paused;

++         }

++         break;

++     }

++     case Paused:

++         // starve the sink so that it goes idle

++         return 0;

++     }

++

++     if (source.isEmpty())

++         return 0;

++

++     memcpy(data, source.data(), source.size());

++     m_bufferSpan = drop(m_bufferSpan, source.size());

++

++     if (m_bufferSpan.isEmpty())

++         fetchMore();

++

++     m_bytesRead += source.size();

++     return source.size();

++ }

++

++ bool AudioSource::atEnd() const

++ {

++     // not done as long as QIODevice's buffer is not empty

++     if (!QIODevice::atEnd() && QIODevice::bytesAvailable())

++         return false;

++

++     // If we get here, bytesAvailable() has returned 0, so our buffers are

++     // exhaused. Try to see if we are waiting for readOperation to finish.

++     AsyncStatus status = AsyncStatus::Completed;

++     if (readOperation) {

++         ComPtr<IAsyncInfo> asyncInfo;

++         if (HRESULT hr = readOperation.As(&asyncInfo); SUCCEEDED(hr))

++             asyncInfo->get_Status(&status);

++     }

++     if (status == AsyncStatus::Started)

++         return false;

++

++     // ... or if there is more in the stream

++     UINT64 ioPos = 0;

++     UINT64 ioSize = 0;

++     if (randomAccessStream) {

++         randomAccessStream->get_Size(&ioSize);

++         randomAccessStream->get_Position(&ioPos);

++     }

++     return ioPos >= ioSize;

++ }

++

++ /*

++     Completion handler for synthesising the stream.

++

++     Is called as soon as synthesized pcm data is available, at which point we can

++     open the QIODevice, fetch data, and start the audio sink.

++ */

++ HRESULT AudioSource::Invoke(IAsyncOperation<SpeechSynthesisStream*> *operation, AsyncStatus status)

++ {

++     Q_ASSERT(operation == synthOperation.Get());

++     ComPtr<IAsyncInfo> asyncInfo;

++     synthOperation.As(&asyncInfo);

++

++     if (status == AsyncStatus::Error) {

++         QString errorString;

++         if (asyncInfo) {

++             HRESULT errorCode;

++             asyncInfo->get_ErrorCode(&errorCode);

++             if (errorCode == 0x80131537) // Windows gives us only an Unknown error

++                 errorString = QStringLiteral("Error when synthesizing: Input format error");

++             else

++                 errorString = QSystemError(errorCode, QSystemError::NativeError).toString();

++         } else {

++             errorString = QStringLiteral("Error when synthesizing: no information available");

++         }

++         setErrorString(errorString);

++         emit errorInStream();

++     }

++     if (status != AsyncStatus::Completed) {

++         if (asyncInfo)

++             asyncInfo->Close();

++         synthOperation.Reset();

++

++         return E_FAIL;

++     }

++

++

++     ComPtr<ISpeechSynthesisStream> speechStream;

++     HRESULT hr = operation->GetResults(&speechStream);

++     RETURN_HR_IF_FAILED("Could not access stream.");

++

++     hr = speechStream.As(&inputStream);

++     RETURN_HR_IF_FAILED("Could not cast to inputStream.");

++     inputStream.As(&randomAccessStream);

++

++     ComPtr<IBufferFactory> bufferFactory;

++     hr = RoGetActivationFactory(HString::MakeReference(RuntimeClass_Windows_Storage_Streams_Buffer).Get(),

++                                 IID_PPV_ARGS(&bufferFactory));

++     RETURN_HR_IF_FAILED("Could not create buffer factory.");

++     // use the same buffer size as default read chunk size

++     bufferFactory->Create(16384, m_buffer.GetAddressOf());

++

++     hr = m_buffer->QueryInterface(IID_PPV_ARGS(&bufferByteAccess));

++     RETURN_HR_IF_FAILED("Could not access buffer.");

++

++     populateBoundaries();

++

++     // release our reference to the speech stream operation

++     if (asyncInfo)

++         asyncInfo->Close();

++     synthOperation.Reset();

++

++     // we are buffered, but we don't want QIODevice to buffer as well

++     open(QIODevice::ReadOnly|QIODevice::Unbuffered);

++     emit streamReady(audioFormat);

++     fetchMore();

++     return S_OK;

++ }

++

++ /*

++ */

++ void AudioSource::populateBoundaries()

++ {

++     ComPtr<ITimedMetadataTrackProvider> metaData;

++     if (!SUCCEEDED(inputStream.As(&metaData)))

++         return;

++

++     ComPtr<IVectorView<TimedMetadataTrack*>> metaDataTracks;

++     metaData->get_TimedMetadataTracks(&metaDataTracks);

++     quint32 trackCount = 0;

++     metaDataTracks->get_Size(&trackCount);

++

++     for (quint32 i = 0; i < trackCount; ++i) {

++         ComPtr<ITimedMetadataTrack> metaDataTrack;

++         HRESULT hr = metaDataTracks->GetAt(i, &metaDataTrack);

++         Q_ASSERT_SUCCEEDED(hr);

++

++         const auto boundaryType = [metaDataTrack]{

++             ComPtr<IMediaTrack> mediaTrack;

++             HRESULT hr = metaDataTrack.As(&mediaTrack);

++             Q_ASSERT_SUCCEEDED(hr);

++

++             if (HString hstr; SUCCEEDED(mediaTrack->get_Id(hstr.GetAddressOf()))) {

++                 const QString trackName = QString::fromWCharArray(hstr.GetRawBuffer(0));

++                 if (trackName == QStringLiteral("SpeechWord"))

++                     return Boundary::Word;

++                 if (trackName == QStringLiteral("SpeechSentence"))

++                     return Boundary::Sentence;

++             }

++             return Boundary::Unknown;

++         }();

++         if (boundaryType == Boundary::Unknown)

++             continue;

++

++         ComPtr<IVectorView<IMediaCue*>> cues;

++         if (!SUCCEEDED(metaDataTrack->get_Cues(&cues)))

++             continue;

++

++         quint32 cueCount = 0;

++         cues->get_Size(&cueCount);

++         boundaries.reserve(boundaries.size() + cueCount);

++

++         for (quint32 j = 0; j < cueCount; ++j) {

++             ComPtr<IMediaCue> cue;

++             hr = cues->GetAt(j, &cue);

++             Q_ASSERT_SUCCEEDED(hr);

++

++             ComPtr<ISpeechCue> speechCue;

++             if (!SUCCEEDED(cue.As(&speechCue))) {

++                 qWarning("Invalid cue");

++                 break;

++             }

++

++             QString text;

++             if (HString hstr; SUCCEEDED(speechCue->get_Text(hstr.GetAddressOf())))

++                 text = QString::fromWCharArray(hstr.GetRawBuffer(0));

++             int startIndex = -1;

++             if (IReference<int> *refInt; SUCCEEDED(speechCue->get_StartPositionInInput(&refInt)))

++                 refInt->get_Value(&startIndex);

++             int endIndex = -1;

++             if (IReference<int> *refInt; SUCCEEDED(speechCue->get_EndPositionInInput(&refInt)))

++                 refInt->get_Value(&endIndex);

++

++             // A time period expressed in 100-nanosecond units. the Duration property is always 0 for speech.

++             TimeSpan startTime = {};

++             cue->get_StartTime(&startTime);

++             const qint64 usec = startTime.Duration / 10; // QAudioSink APIs operate on microseconds

++             boundaries.append(Boundary{boundaryType, text, startIndex, endIndex, usec});

++         }

++     }

++     std::sort(boundaries.begin(), boundaries.end());

++ }

++

++ /*

++     Completion handler for reading from the stream.

++

++     Resets the COM buffer so that it points at the correct position in the

++     stream, and emits readyRead so that the sink pulls more data.

++ */

++ HRESULT AudioSource::Invoke(IAsyncOperationWithProgress<IBuffer*, unsigned int> *read,

++                             AsyncStatus status)

++ {

++     if (status != AsyncStatus::Completed)

++         return E_FAIL;

++

++     // there should never be multiple read operations

++     Q_ASSERT(readOperation.Get() == read);

++

++     HRESULT hr = read->GetResults(&m_buffer);

++     RETURN_HR_IF_FAILED("Could not access buffer.");

++

++     m_bufferSpan = [&] {

++         byte *ptr = nullptr;

++         bufferByteAccess->Buffer(&ptr);

++         UINT32 length = 0;

++         m_buffer->get_Length(&length);

++         return QSpan(reinterpret_cast<const char *>(ptr), qsizetype(length));

++     }();

++

++     // Skip the RIFF header in the first buffer to prevent an audible click.

++     constexpr qsizetype WaveHeaderLength = 44;

++     if (!m_headerConsumed) {

++         Q_ASSERT(m_bytesRead == 0);

++         Q_ASSERT(m_bufferSpan.size() >= WaveHeaderLength);

++         Q_ASSERT(QByteArrayView(m_bufferSpan).startsWith("RIFF"));

++         m_bufferSpan = m_bufferSpan.subspan(WaveHeaderLength);

++         m_headerConsumed = true;

++     }

++

++     ComPtr<IAsyncInfo> asyncInfo;

++     if (HRESULT hr = readOperation.As(&asyncInfo); SUCCEEDED(hr))

++         asyncInfo->Close();

++     readOperation.Reset();

++

++     // inform the sink that more data has arrived

++     if (m_pause == NoPause && !m_bufferSpan.isEmpty())

++         emit readyRead();

++

++     return S_OK;

++ }

++

++ /*

++     Starts an asynchronous read operation. There can only be one such

++     operation pending at any given time, so fetchMore must only be called

++     if the buffer provided by a previous operation is exhaused.

++ */

++ bool AudioSource::fetchMore()

++ {

++     Q_ASSERT(m_buffer);

++

++     if (readOperation) {

++         qWarning () << "Fetching more while a read operation is already pending";

++         return false;

++     }

++

++     UINT32 capacity;

++     m_buffer->get_Capacity(&capacity);

++     InputStreamOptions streamOptions = {};

++     HRESULT hr = inputStream->ReadAsync(m_buffer.Get(), capacity, streamOptions, readOperation.GetAddressOf());

++     if (!SUCCEEDED(hr))

++         return false;

++

++     readOperation->put_Completed(this);

++     return true;

++ }

++

++ QT_END_NAMESPACE

Revision control