qffmpegwindowcapture_uwp.cpp

++ // Copyright (C) 2022 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only

++

++ #include "qffmpegwindowcapture_uwp_p.h"

++ #include <QtMultimedia/private/qsurfacecapturegrabber_p.h>

++

++ #include <QtCore/qloggingcategory.h>

++ #include <QtCore/qscopeguard.h>

++ #include <QtCore/qthread.h>

++ #include <QtCore/private/qexpected_p.h>

++ #include <QtCore/private/qfactorycacheregistration_p.h>

++ #include <QtCore/private/qsystemerror_p.h>

++ #include <QtGui/qguiapplication.h>

++ #include <QtGui/qwindow.h>

++ #include <QtGui/qpa/qplatformscreen_p.h>

++ #include <QtMultimedia/qabstractvideobuffer.h>

++ #include <QtMultimedia/qvideoframe.h>

++ #include <QtMultimedia/private/qcapturablewindow_p.h>

++ #include <QtMultimedia/private/qmultimediautils_p.h>

++ #include <QtMultimedia/private/qvideoframe_p.h>

++

++ #include <unknwn.h>

++ #include <winrt/base.h>

++ // Workaround for Windows SDK bug.

++ // See https://github.com/microsoft/Windows.UI.Composition-Win32-Samples/issues/47

++ namespace winrt::impl

++ {

++ template <typename Async>

++ auto wait_for(Async const& async, Windows::Foundation::TimeSpan const& timeout);

++ }

++ #include <winrt/Windows.Foundation.Collections.h>

++ #include <winrt/Windows.Graphics.Capture.h>

++ #include <winrt/Windows.Graphics.DirectX.h>

++ #include <winrt/Windows.Graphics.DirectX.Direct3D11.h>

++ #include <Windows.Graphics.Capture.h>

++ #include <Windows.Graphics.Capture.Interop.h>

++ #include <windows.graphics.directx.direct3d11.interop.h>

++

++ #include <d3d11.h>

++ #include <dwmapi.h>

++ #include <lowlevelmonitorconfigurationapi.h>

++ #include <physicalmonitorenumerationapi.h>

++

++ #include <memory>

++ #include <system_error>

++

++ QT_BEGIN_NAMESPACE

++

++ using namespace Qt::StringLiterals;

++

++ using namespace winrt::Windows::Graphics::Capture;

++ using namespace winrt::Windows::Graphics::DirectX;

++ using namespace winrt::Windows::Graphics::DirectX::Direct3D11;

++ using namespace Windows::Graphics::DirectX::Direct3D11;

++

++ using winrt::check_hresult;

++ using winrt::com_ptr;

++ using winrt::guid_of;

++

++ namespace {

++

++ Q_LOGGING_CATEGORY(qLcWindowCaptureUwp, "qt.multimedia.ffmpeg.windowcapture.uwp");

++

++ winrt::Windows::Graphics::SizeInt32 getWindowSize(HWND hwnd)

++ {

++     RECT windowRect{};

++     ::GetWindowRect(hwnd, &windowRect);

++

++     return { windowRect.right - windowRect.left, windowRect.bottom - windowRect.top };

++ }

++

++ QSize asQSize(winrt::Windows::Graphics::SizeInt32 size)

++ {

++     return { size.Width, size.Height };

++ }

++

++ struct MultithreadedApartment

++ {

++     MultithreadedApartment(const MultithreadedApartment &) = delete;

++     MultithreadedApartment &operator=(const MultithreadedApartment &) = delete;

++

++     MultithreadedApartment() { winrt::init_apartment(); }

++     ~MultithreadedApartment() { winrt::uninit_apartment(); }

++ };

++

++ class QUwpTextureVideoBuffer : public QAbstractVideoBuffer

++ {

++ public:

++     QUwpTextureVideoBuffer(com_ptr<IDXGISurface> &&surface) : m_surface(surface) { }

++

++     ~QUwpTextureVideoBuffer() override { Q_ASSERT(m_mapMode == QVideoFrame::NotMapped); }

++

++     MapData map(QVideoFrame::MapMode mode) override

++     {

++         if (m_mapMode != QVideoFrame::NotMapped)

++             return {};

++

++         if (mode == QVideoFrame::ReadOnly) {

++             DXGI_MAPPED_RECT rect = {};

++             HRESULT hr = m_surface->Map(&rect, DXGI_MAP_READ);

++             if (SUCCEEDED(hr)) {

++                 DXGI_SURFACE_DESC desc = {};

++                 hr = m_surface->GetDesc(&desc);

++

++                 MapData md = {};

++                 md.planeCount = 1;

++                 md.bytesPerLine[0] = rect.Pitch;

++                 md.data[0] = rect.pBits;

++                 md.dataSize[0] = rect.Pitch * desc.Height;

++

++                 m_mapMode = QVideoFrame::ReadOnly;

++

++                 return md;

++             } else {

++                 qCDebug(qLcWindowCaptureUwp)

++                         << "Failed to map DXGI surface" << QSystemError::windowsComString(hr);

++                 return {};

++             }

++         }

++

++         return {};

++     }

++

++     void unmap() override

++     {

++         if (m_mapMode == QVideoFrame::NotMapped)

++             return;

++

++         const HRESULT hr = m_surface->Unmap();

++         if (FAILED(hr))

++             qCDebug(qLcWindowCaptureUwp)

++                     << "Failed to unmap surface" << QSystemError::windowsComString(hr);

++

++         m_mapMode = QVideoFrame::NotMapped;

++     }

++

++     QVideoFrameFormat format() const override { return {}; }

++

++ private:

++     QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped;

++     com_ptr<IDXGISurface> m_surface;

++ };

++

++ struct WindowGrabber

++ {

++     WindowGrabber() = default;

++

++     WindowGrabber(IDXGIAdapter1 *adapter, HWND hwnd, bool ignoreCursor)

++         : m_captureWindow{ hwnd }

++     {

++         check_hresult(D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, nullptr, 0, nullptr, 0,

++                                         D3D11_SDK_VERSION, m_device.put(), nullptr, nullptr));

++

++         const auto captureItem = createCaptureItem(hwnd);

++

++         // The capture item excludes invisible borders that GetWindowRect includes. Using the

++         // window rect here would make the first frame look like a resize, which is dropped,

++         // and a static window never delivers another one.

++         m_frameSize = captureItem.Size();

++

++         m_framePool = Direct3D11CaptureFramePool::CreateFreeThreaded(

++                 getCaptureDevice(m_device), m_pixelFormat, 1,

++                 captureItem.Size());

++

++         m_session = m_framePool.CreateCaptureSession(captureItem);

++

++         // If supported, capture the cursor unless asked not to

++         if (const auto session2 = m_session.try_as<IGraphicsCaptureSession2>())

++             session2.IsCursorCaptureEnabled(!ignoreCursor);

++

++         // If supported, disable colored border around captured window to match other platforms

++         if (const auto session3 = m_session.try_as<IGraphicsCaptureSession3>())

++             session3.IsBorderRequired(false);

++

++         m_session.StartCapture();

++     }

++

++     ~WindowGrabber()

++     {

++         m_framePool.Close();

++         m_session.Close();

++     }

++

++     com_ptr<IDXGISurface> tryGetFrame()

++     {

++         const Direct3D11CaptureFrame frame = m_framePool.TryGetNextFrame();

++         if (!frame) {

++

++             // Stop capture and report failure if window was closed. If we don't stop,

++             // testing shows that either we don't get any frames, or we get blank frames.

++             // Emitting an error will prevent this inconsistent behavior, and makes the

++             // Windows implementation behave like the Linux implementation

++             if (!IsWindow(m_captureWindow))

++                 throw std::runtime_error("Window was closed");

++

++             // Blank frames may come spuriously if no new window texture

++             // is available yet.

++             return {};

++         }

++

++         if (m_frameSize != frame.ContentSize()) {

++             m_frameSize = frame.ContentSize();

++             m_framePool.Recreate(getCaptureDevice(m_device), m_pixelFormat, 1, frame.ContentSize());

++             return {};

++         }

++

++         return copyTexture(m_device, frame.Surface());

++     }

++

++ private:

++     static GraphicsCaptureItem createCaptureItem(HWND hwnd)

++     {

++         const auto factory = winrt::get_activation_factory<GraphicsCaptureItem>();

++         const auto interop = factory.as<IGraphicsCaptureItemInterop>();

++

++         GraphicsCaptureItem item = { nullptr };

++         winrt::hresult status = S_OK;

++

++         // Attempt to create capture item with retry, because this occasionally fails,

++         // particularly in unit tests. When the failure code is E_INVALIDARG, it

++         // seems to help to sleep for a bit and retry. See QTBUG-116025.

++         constexpr int maxRetry = 10;

++         constexpr std::chrono::milliseconds retryDelay{ 100 };

++         for (int retryNum = 0; retryNum < maxRetry; ++retryNum) {

++

++             status = interop->CreateForWindow(hwnd, winrt::guid_of<GraphicsCaptureItem>(),

++                                               winrt::put_abi(item));

++

++             if (status != E_INVALIDARG)

++                 break;

++

++             qCWarning(qLcWindowCaptureUwp)

++                     << "Failed to create capture item:"

++                     << QString::fromStdWString(winrt::hresult_error(status).message().c_str())

++                     << "Retry number" << retryNum;

++

++             if (retryNum + 1 < maxRetry)

++                 QThread::sleep(retryDelay);

++         }

++

++         // Throw if we fail to create the capture item

++         check_hresult(status);

++

++         return item;

++     }

++

++     static IDirect3DDevice getCaptureDevice(const com_ptr<ID3D11Device> &d3dDevice)

++     {

++         const auto dxgiDevice = d3dDevice.as<IDXGIDevice>();

++

++         com_ptr<IInspectable> device;

++         check_hresult(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.get(), device.put()));

++

++         return device.as<IDirect3DDevice>();

++     }

++

++     static com_ptr<IDXGISurface> copyTexture(const com_ptr<ID3D11Device> &device,

++                                              const IDirect3DSurface &capturedTexture)

++     {

++         const auto dxgiInterop{ capturedTexture.as<IDirect3DDxgiInterfaceAccess>() };

++         if (!dxgiInterop)

++             return {};

++

++         com_ptr<IDXGISurface> dxgiSurface;

++         check_hresult(dxgiInterop->GetInterface(guid_of<IDXGISurface>(), dxgiSurface.put_void()));

++

++         DXGI_SURFACE_DESC desc = {};

++         check_hresult(dxgiSurface->GetDesc(&desc));

++

++         D3D11_TEXTURE2D_DESC texDesc = {};

++         texDesc.Width = desc.Width;

++         texDesc.Height = desc.Height;

++         texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;

++         texDesc.Usage = D3D11_USAGE_STAGING;

++         texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;

++         texDesc.MiscFlags = 0;

++         texDesc.BindFlags = 0;

++         texDesc.ArraySize = 1;

++         texDesc.MipLevels = 1;

++         texDesc.SampleDesc = { 1, 0 };

++

++         com_ptr<ID3D11Texture2D> texture;

++         check_hresult(device->CreateTexture2D(&texDesc, nullptr, texture.put()));

++

++         com_ptr<ID3D11DeviceContext> ctx;

++         device->GetImmediateContext(ctx.put());

++         ctx->CopyResource(texture.get(), dxgiSurface.as<ID3D11Resource>().get());

++

++         return texture.as<IDXGISurface>();

++     }

++

++     MultithreadedApartment m_comApartment{};

++     HWND m_captureWindow{};

++     winrt::Windows::Graphics::SizeInt32 m_frameSize{};

++     com_ptr<ID3D11Device> m_device;

++     Direct3D11CaptureFramePool m_framePool{ nullptr };

++     GraphicsCaptureSession m_session{ nullptr };

++     const DirectXPixelFormat m_pixelFormat = DirectXPixelFormat::R8G8B8A8UIntNormalized;

++ };

++

++ } // namespace

++

++ class QFFmpegWindowCaptureUwp::Grabber : public QSurfaceCaptureGrabber

++ {

++ public:

++     Grabber(QFFmpegWindowCaptureUwp &capture, HWND hwnd)

++         : QSurfaceCaptureGrabber(),

++           m_hwnd(hwnd),

++           m_ignoreCursor(capture.ignoreCursor()),

++           m_format(QVideoFrameFormat(asQSize(getWindowSize(hwnd)),

++                                      QVideoFrameFormat::Format_RGBX8888))

++     {

++         const HMONITOR monitor = MonitorFromWindow(hwnd, MONITOR_DEFAULTTONULL);

++         m_adapter = getAdapter(monitor);

++

++         addFrameCallback(&capture, &QFFmpegWindowCaptureUwp::newVideoFrame);

++         connect(this, &Grabber::errorUpdated, &capture, &QFFmpegWindowCaptureUwp::updateError);

++

++         const auto refreshRate = getMonitorRefreshRateHz(monitor);

++         const auto defaultRate = refreshRate ? qMin(*refreshRate, DefaultScreenCaptureFrameRate)

++                                              : DefaultScreenCaptureFrameRate;

++         setFrameRate(capture.frameRate().value_or(defaultRate));

++         m_format.setStreamFrameRate(frameRate());

++     }

++

++     ~Grabber() override { stop(); }

++

++     QVideoFrameFormat frameFormat() const { return m_format; }

++

++ protected:

++

++     void initializeGrabbingContext() override

++     {

++         if (!m_adapter || !IsWindow(m_hwnd))

++             return; // Error already logged

++

++         try {

++             m_windowGrabber = std::make_unique<WindowGrabber>(

++                 m_adapter.get(),

++                 m_hwnd,

++                 m_ignoreCursor);

++

++             QSurfaceCaptureGrabber::initializeGrabbingContext();

++         } catch (const winrt::hresult_error &err) {

++

++             const QString message = QLatin1String("Unable to capture window: ")

++                     + QString::fromWCharArray(err.message().c_str());

++

++             updateError(QPlatformSurfaceCapture::Error::InternalError, message);

++         }

++     }

++

++     void finalizeGrabbingContext() override

++     {

++         QSurfaceCaptureGrabber::finalizeGrabbingContext();

++         m_windowGrabber = nullptr;

++     }

++

++     QVideoFrame grabFrame() override

++     {

++         try {

++             com_ptr<IDXGISurface> texture = m_windowGrabber->tryGetFrame();

++             if (!texture)

++                 return {}; // No frame available yet

++

++             const QSize size = getTextureSize(texture);

++

++             m_format.setFrameSize(size);

++

++             if (m_format.streamFrameRate() != activeFrameRate())

++                 m_format.setStreamFrameRate(activeFrameRate());

++

++             return QVideoFramePrivate::createFrame(

++                     std::make_unique<QUwpTextureVideoBuffer>(std::move(texture)), m_format);

++

++         } catch (const winrt::hresult_error &err) {

++

++             const QString message = QLatin1String("Window capture failed: ")

++                     + QString::fromWCharArray(err.message().c_str());

++

++             updateError(QPlatformSurfaceCapture::Error::InternalError, message);

++         } catch (const std::runtime_error& e) {

++             updateError(QPlatformSurfaceCapture::Error::CaptureFailed, QString::fromLatin1(e.what()));

++         }

++

++         return {};

++     }

++

++ private:

++     static com_ptr<IDXGIAdapter1> getAdapter(HMONITOR handle)

++     {

++         com_ptr<IDXGIFactory1> factory;

++         check_hresult(CreateDXGIFactory1(guid_of<IDXGIFactory1>(), factory.put_void()));

++

++         com_ptr<IDXGIAdapter1> adapter;

++         for (quint32 i = 0; factory->EnumAdapters1(i, adapter.put()) == S_OK; adapter = nullptr, i++) {

++             com_ptr<IDXGIOutput> output;

++             for (quint32 j = 0; adapter->EnumOutputs(j, output.put()) == S_OK; output = nullptr, j++) {

++                 DXGI_OUTPUT_DESC desc = {};

++                 HRESULT hr = output->GetDesc(&desc);

++                 if (hr == S_OK && desc.Monitor == handle)

++                     return adapter;

++             }

++         }

++         return {};

++     }

++

++     static QSize getTextureSize(const com_ptr<IDXGISurface> &surf)

++     {

++         if (!surf)

++             return {};

++

++         DXGI_SURFACE_DESC desc;

++         check_hresult(surf->GetDesc(&desc));

++

++         return { static_cast<int>(desc.Width), static_cast<int>(desc.Height) };

++     }

++

++     static q23::expected<qreal, QString> getMonitorRefreshRateHz(HMONITOR handle)

++     {

++         if (!handle)

++             return q23::unexpected{ u"No monitor"_s };

++

++         DWORD count = 0;

++         if (GetNumberOfPhysicalMonitorsFromHMONITOR(handle, &count)) {

++             std::vector<PHYSICAL_MONITOR> monitors{ count };

++             if (GetPhysicalMonitorsFromHMONITOR(handle, count, monitors.data())) {

++                 const auto cleanup = qScopeGuard([count, &monitors] {

++                     DestroyPhysicalMonitors(count, monitors.data());

++                 });

++                 for (const auto &monitor : std::as_const(monitors)) {

++                     MC_TIMING_REPORT screenTiming = {};

++                     if (GetTimingReport(monitor.hPhysicalMonitor, &screenTiming)) {

++                         // Empirically we found that GetTimingReport does not return

++                         // the frequency in updates per second as documented, but in

++                         // updates per 100 seconds.

++                         return static_cast<qreal>(screenTiming.dwVerticalFrequencyInHZ) / 100.0;

++                     }

++                 }

++             }

++         }

++         return q23::unexpected{ u"Couldn't get monitor refresh rate"_s };

++     }

++

++     HWND m_hwnd{};

++     bool m_ignoreCursor = false;

++     com_ptr<IDXGIAdapter1> m_adapter{};

++     std::unique_ptr<WindowGrabber> m_windowGrabber;

++     QVideoFrameFormat m_format;

++ };

++

++ QFFmpegWindowCaptureUwp::QFFmpegWindowCaptureUwp() : QPlatformSurfaceCapture(WindowSource{})

++ {

++     qCDebug(qLcWindowCaptureUwp) << "Creating UWP screen capture";

++ }

++

++ QFFmpegWindowCaptureUwp::~QFFmpegWindowCaptureUwp() = default;

++

++ static QString isCapturableWindow(HWND hwnd)

++ {

++     if (!IsWindow(hwnd))

++         return u"Invalid window handle"_s;

++

++     if (hwnd == GetShellWindow())

++         return u"Cannot capture the shell window"_s;

++

++     wchar_t className[MAX_PATH] = {};

++     GetClassName(hwnd, className, MAX_PATH);

++     if (QString::fromWCharArray(className).length() == 0)

++         return u"Cannot capture windows without a class name"_s;

++

++     if (!IsWindowVisible(hwnd))

++         return u"Cannot capture invisible windows"_s;

++

++     if (GetAncestor(hwnd, GA_ROOT) != hwnd)

++         return u"Can only capture root windows"_s;

++

++     const LONG_PTR style = GetWindowLongPtr(hwnd, GWL_STYLE);

++     if (style & WS_DISABLED)

++         return u"Cannot capture disabled windows"_s;

++

++     const LONG_PTR exStyle = GetWindowLongPtr(hwnd, GWL_EXSTYLE);

++     if (exStyle & WS_EX_TOOLWINDOW)

++         return u"No tooltips"_s;

++

++     DWORD cloaked = FALSE;

++     const HRESULT hr = DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, &cloaked, sizeof(cloaked));

++     if (SUCCEEDED(hr) && cloaked == DWM_CLOAKED_SHELL)

++         return u"Cannot capture cloaked windows"_s;

++

++     return {};

++ }

++

++ bool QFFmpegWindowCaptureUwp::setActiveInternal(bool active)

++ {

++     if (static_cast<bool>(m_grabber) == active)

++         return false;

++

++     if (m_grabber) {

++         m_grabber.reset();

++         return true;

++     }

++

++     const auto window = source<WindowSource>();

++     const auto handle = QCapturableWindowPrivate::handle(window);

++

++     const auto hwnd = reinterpret_cast<HWND>(handle ? handle->id : 0);

++     if (const QString error = isCapturableWindow(hwnd); !error.isEmpty()) {

++         updateError(QPlatformSurfaceCapture::Error::InternalError, error);

++         return false;

++     }

++

++     m_grabber = std::make_unique<Grabber>(*this, hwnd);

++     m_grabber->start();

++

++     return true;

++ }

++

++ bool QFFmpegWindowCaptureUwp::isSupported()

++ {

++     // It has been observed that GraphicsCaptureSession::IsSupported()

++     // may throw under certain environments, such as when running

++     // Qt in a headless Windows environment.

++     try {

++         return GraphicsCaptureSession::IsSupported();

++     } catch (const winrt::hresult_error &err) {

++         qCWarning(qLcWindowCaptureUwp)

++             << "GraphicsCaptureSession::IsSupported() threw exception:"

++             << QString::fromWCharArray(err.message().c_str());

++         return false;

++     } catch (const std::exception &err) {

++         qCWarning(qLcWindowCaptureUwp)

++             << "GraphicsCaptureSession::IsSupported() failed, threw exception:"

++             << err.what();

++         return false;

++     }

++ }

++

++ QVideoFrameFormat QFFmpegWindowCaptureUwp::frameFormat() const

++ {

++     if (m_grabber)

++         return m_grabber->frameFormat();

++     return {};

++ }

++

++ QT_END_NAMESPACE

Revision control