Revision control
Copy as Markdown
// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegwindowcapture_uwp_p.h"
#include <QtMultimedia/private/qsurfacecapturegrabber_p.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qscopeguard.h>
#include <QtCore/qthread.h>
#include <QtCore/private/qexpected_p.h>
#include <QtCore/private/qfactorycacheregistration_p.h>
#include <QtCore/private/qsystemerror_p.h>
#include <QtGui/qguiapplication.h>
#include <QtGui/qwindow.h>
#include <QtGui/qpa/qplatformscreen_p.h>
#include <QtMultimedia/qabstractvideobuffer.h>
#include <QtMultimedia/qvideoframe.h>
#include <QtMultimedia/private/qcapturablewindow_p.h>
#include <QtMultimedia/private/qmultimediautils_p.h>
#include <QtMultimedia/private/qvideoframe_p.h>
#include <unknwn.h>
#include <winrt/base.h>
// Workaround for Windows SDK bug.
namespace winrt::impl
{
template <typename Async>
auto wait_for(Async const& async, Windows::Foundation::TimeSpan const& timeout);
}
#include <winrt/Windows.Foundation.Collections.h>
#include <winrt/Windows.Graphics.Capture.h>
#include <winrt/Windows.Graphics.DirectX.h>
#include <winrt/Windows.Graphics.DirectX.Direct3D11.h>
#include <Windows.Graphics.Capture.h>
#include <Windows.Graphics.Capture.Interop.h>
#include <windows.graphics.directx.direct3d11.interop.h>
#include <d3d11.h>
#include <dwmapi.h>
#include <lowlevelmonitorconfigurationapi.h>
#include <physicalmonitorenumerationapi.h>
#include <memory>
#include <system_error>
QT_BEGIN_NAMESPACE
using namespace Qt::StringLiterals;
using namespace winrt::Windows::Graphics::Capture;
using namespace winrt::Windows::Graphics::DirectX;
using namespace winrt::Windows::Graphics::DirectX::Direct3D11;
using namespace Windows::Graphics::DirectX::Direct3D11;
using winrt::check_hresult;
using winrt::com_ptr;
using winrt::guid_of;
namespace {
Q_LOGGING_CATEGORY(qLcWindowCaptureUwp, "qt.multimedia.ffmpeg.windowcapture.uwp");
winrt::Windows::Graphics::SizeInt32 getWindowSize(HWND hwnd)
{
RECT windowRect{};
::GetWindowRect(hwnd, &windowRect);
return { windowRect.right - windowRect.left, windowRect.bottom - windowRect.top };
}
QSize asQSize(winrt::Windows::Graphics::SizeInt32 size)
{
return { size.Width, size.Height };
}
struct MultithreadedApartment
{
MultithreadedApartment(const MultithreadedApartment &) = delete;
MultithreadedApartment &operator=(const MultithreadedApartment &) = delete;
MultithreadedApartment() { winrt::init_apartment(); }
~MultithreadedApartment() { winrt::uninit_apartment(); }
};
class QUwpTextureVideoBuffer : public QAbstractVideoBuffer
{
public:
QUwpTextureVideoBuffer(com_ptr<IDXGISurface> &&surface) : m_surface(surface) { }
~QUwpTextureVideoBuffer() override { Q_ASSERT(m_mapMode == QVideoFrame::NotMapped); }
MapData map(QVideoFrame::MapMode mode) override
{
if (m_mapMode != QVideoFrame::NotMapped)
return {};
if (mode == QVideoFrame::ReadOnly) {
DXGI_MAPPED_RECT rect = {};
HRESULT hr = m_surface->Map(&rect, DXGI_MAP_READ);
if (SUCCEEDED(hr)) {
DXGI_SURFACE_DESC desc = {};
hr = m_surface->GetDesc(&desc);
MapData md = {};
md.planeCount = 1;
md.bytesPerLine[0] = rect.Pitch;
md.data[0] = rect.pBits;
md.dataSize[0] = rect.Pitch * desc.Height;
m_mapMode = QVideoFrame::ReadOnly;
return md;
} else {
qCDebug(qLcWindowCaptureUwp)
<< "Failed to map DXGI surface" << QSystemError::windowsComString(hr);
return {};
}
}
return {};
}
void unmap() override
{
if (m_mapMode == QVideoFrame::NotMapped)
return;
const HRESULT hr = m_surface->Unmap();
if (FAILED(hr))
qCDebug(qLcWindowCaptureUwp)
<< "Failed to unmap surface" << QSystemError::windowsComString(hr);
m_mapMode = QVideoFrame::NotMapped;
}
QVideoFrameFormat format() const override { return {}; }
private:
QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped;
com_ptr<IDXGISurface> m_surface;
};
struct WindowGrabber
{
WindowGrabber() = default;
WindowGrabber(IDXGIAdapter1 *adapter, HWND hwnd, bool ignoreCursor)
: m_captureWindow{ hwnd }
{
check_hresult(D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, nullptr, 0, nullptr, 0,
D3D11_SDK_VERSION, m_device.put(), nullptr, nullptr));
const auto captureItem = createCaptureItem(hwnd);
// The capture item excludes invisible borders that GetWindowRect includes. Using the
// window rect here would make the first frame look like a resize, which is dropped,
// and a static window never delivers another one.
m_frameSize = captureItem.Size();
m_framePool = Direct3D11CaptureFramePool::CreateFreeThreaded(
getCaptureDevice(m_device), m_pixelFormat, 1,
captureItem.Size());
m_session = m_framePool.CreateCaptureSession(captureItem);
// If supported, capture the cursor unless asked not to
if (const auto session2 = m_session.try_as<IGraphicsCaptureSession2>())
session2.IsCursorCaptureEnabled(!ignoreCursor);
// If supported, disable colored border around captured window to match other platforms
if (const auto session3 = m_session.try_as<IGraphicsCaptureSession3>())
session3.IsBorderRequired(false);
m_session.StartCapture();
}
~WindowGrabber()
{
m_framePool.Close();
m_session.Close();
}
com_ptr<IDXGISurface> tryGetFrame()
{
const Direct3D11CaptureFrame frame = m_framePool.TryGetNextFrame();
if (!frame) {
// Stop capture and report failure if window was closed. If we don't stop,
// testing shows that either we don't get any frames, or we get blank frames.
// Emitting an error will prevent this inconsistent behavior, and makes the
// Windows implementation behave like the Linux implementation
if (!IsWindow(m_captureWindow))
throw std::runtime_error("Window was closed");
// Blank frames may come spuriously if no new window texture
// is available yet.
return {};
}
if (m_frameSize != frame.ContentSize()) {
m_frameSize = frame.ContentSize();
m_framePool.Recreate(getCaptureDevice(m_device), m_pixelFormat, 1, frame.ContentSize());
return {};
}
return copyTexture(m_device, frame.Surface());
}
private:
static GraphicsCaptureItem createCaptureItem(HWND hwnd)
{
const auto factory = winrt::get_activation_factory<GraphicsCaptureItem>();
const auto interop = factory.as<IGraphicsCaptureItemInterop>();
GraphicsCaptureItem item = { nullptr };
winrt::hresult status = S_OK;
// Attempt to create capture item with retry, because this occasionally fails,
// particularly in unit tests. When the failure code is E_INVALIDARG, it
// seems to help to sleep for a bit and retry. See QTBUG-116025.
constexpr int maxRetry = 10;
constexpr std::chrono::milliseconds retryDelay{ 100 };
for (int retryNum = 0; retryNum < maxRetry; ++retryNum) {
status = interop->CreateForWindow(hwnd, winrt::guid_of<GraphicsCaptureItem>(),
winrt::put_abi(item));
if (status != E_INVALIDARG)
break;
qCWarning(qLcWindowCaptureUwp)
<< "Failed to create capture item:"
<< QString::fromStdWString(winrt::hresult_error(status).message().c_str())
<< "Retry number" << retryNum;
if (retryNum + 1 < maxRetry)
QThread::sleep(retryDelay);
}
// Throw if we fail to create the capture item
check_hresult(status);
return item;
}
static IDirect3DDevice getCaptureDevice(const com_ptr<ID3D11Device> &d3dDevice)
{
const auto dxgiDevice = d3dDevice.as<IDXGIDevice>();
com_ptr<IInspectable> device;
check_hresult(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.get(), device.put()));
return device.as<IDirect3DDevice>();
}
static com_ptr<IDXGISurface> copyTexture(const com_ptr<ID3D11Device> &device,
const IDirect3DSurface &capturedTexture)
{
const auto dxgiInterop{ capturedTexture.as<IDirect3DDxgiInterfaceAccess>() };
if (!dxgiInterop)
return {};
com_ptr<IDXGISurface> dxgiSurface;
check_hresult(dxgiInterop->GetInterface(guid_of<IDXGISurface>(), dxgiSurface.put_void()));
DXGI_SURFACE_DESC desc = {};
check_hresult(dxgiSurface->GetDesc(&desc));
D3D11_TEXTURE2D_DESC texDesc = {};
texDesc.Width = desc.Width;
texDesc.Height = desc.Height;
texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
texDesc.Usage = D3D11_USAGE_STAGING;
texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
texDesc.MiscFlags = 0;
texDesc.BindFlags = 0;
texDesc.ArraySize = 1;
texDesc.MipLevels = 1;
texDesc.SampleDesc = { 1, 0 };
com_ptr<ID3D11Texture2D> texture;
check_hresult(device->CreateTexture2D(&texDesc, nullptr, texture.put()));
com_ptr<ID3D11DeviceContext> ctx;
device->GetImmediateContext(ctx.put());
ctx->CopyResource(texture.get(), dxgiSurface.as<ID3D11Resource>().get());
return texture.as<IDXGISurface>();
}
MultithreadedApartment m_comApartment{};
HWND m_captureWindow{};
winrt::Windows::Graphics::SizeInt32 m_frameSize{};
com_ptr<ID3D11Device> m_device;
Direct3D11CaptureFramePool m_framePool{ nullptr };
GraphicsCaptureSession m_session{ nullptr };
const DirectXPixelFormat m_pixelFormat = DirectXPixelFormat::R8G8B8A8UIntNormalized;
};
} // namespace
class QFFmpegWindowCaptureUwp::Grabber : public QSurfaceCaptureGrabber
{
public:
Grabber(QFFmpegWindowCaptureUwp &capture, HWND hwnd)
: QSurfaceCaptureGrabber(),
m_hwnd(hwnd),
m_ignoreCursor(capture.ignoreCursor()),
m_format(QVideoFrameFormat(asQSize(getWindowSize(hwnd)),
QVideoFrameFormat::Format_RGBX8888))
{
const HMONITOR monitor = MonitorFromWindow(hwnd, MONITOR_DEFAULTTONULL);
m_adapter = getAdapter(monitor);
addFrameCallback(&capture, &QFFmpegWindowCaptureUwp::newVideoFrame);
connect(this, &Grabber::errorUpdated, &capture, &QFFmpegWindowCaptureUwp::updateError);
const auto refreshRate = getMonitorRefreshRateHz(monitor);
const auto defaultRate = refreshRate ? qMin(*refreshRate, DefaultScreenCaptureFrameRate)
: DefaultScreenCaptureFrameRate;
setFrameRate(capture.frameRate().value_or(defaultRate));
m_format.setStreamFrameRate(frameRate());
}
~Grabber() override { stop(); }
QVideoFrameFormat frameFormat() const { return m_format; }
protected:
void initializeGrabbingContext() override
{
if (!m_adapter || !IsWindow(m_hwnd))
return; // Error already logged
try {
m_windowGrabber = std::make_unique<WindowGrabber>(
m_adapter.get(),
m_hwnd,
m_ignoreCursor);
QSurfaceCaptureGrabber::initializeGrabbingContext();
} catch (const winrt::hresult_error &err) {
const QString message = QLatin1String("Unable to capture window: ")
+ QString::fromWCharArray(err.message().c_str());
updateError(QPlatformSurfaceCapture::Error::InternalError, message);
}
}
void finalizeGrabbingContext() override
{
QSurfaceCaptureGrabber::finalizeGrabbingContext();
m_windowGrabber = nullptr;
}
QVideoFrame grabFrame() override
{
try {
com_ptr<IDXGISurface> texture = m_windowGrabber->tryGetFrame();
if (!texture)
return {}; // No frame available yet
const QSize size = getTextureSize(texture);
m_format.setFrameSize(size);
if (m_format.streamFrameRate() != activeFrameRate())
m_format.setStreamFrameRate(activeFrameRate());
return QVideoFramePrivate::createFrame(
std::make_unique<QUwpTextureVideoBuffer>(std::move(texture)), m_format);
} catch (const winrt::hresult_error &err) {
const QString message = QLatin1String("Window capture failed: ")
+ QString::fromWCharArray(err.message().c_str());
updateError(QPlatformSurfaceCapture::Error::InternalError, message);
} catch (const std::runtime_error& e) {
updateError(QPlatformSurfaceCapture::Error::CaptureFailed, QString::fromLatin1(e.what()));
}
return {};
}
private:
static com_ptr<IDXGIAdapter1> getAdapter(HMONITOR handle)
{
com_ptr<IDXGIFactory1> factory;
check_hresult(CreateDXGIFactory1(guid_of<IDXGIFactory1>(), factory.put_void()));
com_ptr<IDXGIAdapter1> adapter;
for (quint32 i = 0; factory->EnumAdapters1(i, adapter.put()) == S_OK; adapter = nullptr, i++) {
com_ptr<IDXGIOutput> output;
for (quint32 j = 0; adapter->EnumOutputs(j, output.put()) == S_OK; output = nullptr, j++) {
DXGI_OUTPUT_DESC desc = {};
HRESULT hr = output->GetDesc(&desc);
if (hr == S_OK && desc.Monitor == handle)
return adapter;
}
}
return {};
}
static QSize getTextureSize(const com_ptr<IDXGISurface> &surf)
{
if (!surf)
return {};
DXGI_SURFACE_DESC desc;
check_hresult(surf->GetDesc(&desc));
return { static_cast<int>(desc.Width), static_cast<int>(desc.Height) };
}
static q23::expected<qreal, QString> getMonitorRefreshRateHz(HMONITOR handle)
{
if (!handle)
return q23::unexpected{ u"No monitor"_s };
DWORD count = 0;
if (GetNumberOfPhysicalMonitorsFromHMONITOR(handle, &count)) {
std::vector<PHYSICAL_MONITOR> monitors{ count };
if (GetPhysicalMonitorsFromHMONITOR(handle, count, monitors.data())) {
const auto cleanup = qScopeGuard([count, &monitors] {
DestroyPhysicalMonitors(count, monitors.data());
});
for (const auto &monitor : std::as_const(monitors)) {
MC_TIMING_REPORT screenTiming = {};
if (GetTimingReport(monitor.hPhysicalMonitor, &screenTiming)) {
// Empirically we found that GetTimingReport does not return
// the frequency in updates per second as documented, but in
// updates per 100 seconds.
return static_cast<qreal>(screenTiming.dwVerticalFrequencyInHZ) / 100.0;
}
}
}
}
return q23::unexpected{ u"Couldn't get monitor refresh rate"_s };
}
HWND m_hwnd{};
bool m_ignoreCursor = false;
com_ptr<IDXGIAdapter1> m_adapter{};
std::unique_ptr<WindowGrabber> m_windowGrabber;
QVideoFrameFormat m_format;
};
QFFmpegWindowCaptureUwp::QFFmpegWindowCaptureUwp() : QPlatformSurfaceCapture(WindowSource{})
{
qCDebug(qLcWindowCaptureUwp) << "Creating UWP screen capture";
}
QFFmpegWindowCaptureUwp::~QFFmpegWindowCaptureUwp() = default;
static QString isCapturableWindow(HWND hwnd)
{
if (!IsWindow(hwnd))
return u"Invalid window handle"_s;
if (hwnd == GetShellWindow())
return u"Cannot capture the shell window"_s;
wchar_t className[MAX_PATH] = {};
GetClassName(hwnd, className, MAX_PATH);
if (QString::fromWCharArray(className).length() == 0)
return u"Cannot capture windows without a class name"_s;
if (!IsWindowVisible(hwnd))
return u"Cannot capture invisible windows"_s;
if (GetAncestor(hwnd, GA_ROOT) != hwnd)
return u"Can only capture root windows"_s;
const LONG_PTR style = GetWindowLongPtr(hwnd, GWL_STYLE);
if (style & WS_DISABLED)
return u"Cannot capture disabled windows"_s;
const LONG_PTR exStyle = GetWindowLongPtr(hwnd, GWL_EXSTYLE);
if (exStyle & WS_EX_TOOLWINDOW)
return u"No tooltips"_s;
DWORD cloaked = FALSE;
const HRESULT hr = DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, &cloaked, sizeof(cloaked));
if (SUCCEEDED(hr) && cloaked == DWM_CLOAKED_SHELL)
return u"Cannot capture cloaked windows"_s;
return {};
}
bool QFFmpegWindowCaptureUwp::setActiveInternal(bool active)
{
if (static_cast<bool>(m_grabber) == active)
return false;
if (m_grabber) {
m_grabber.reset();
return true;
}
const auto window = source<WindowSource>();
const auto handle = QCapturableWindowPrivate::handle(window);
const auto hwnd = reinterpret_cast<HWND>(handle ? handle->id : 0);
if (const QString error = isCapturableWindow(hwnd); !error.isEmpty()) {
updateError(QPlatformSurfaceCapture::Error::InternalError, error);
return false;
}
m_grabber = std::make_unique<Grabber>(*this, hwnd);
m_grabber->start();
return true;
}
bool QFFmpegWindowCaptureUwp::isSupported()
{
// It has been observed that GraphicsCaptureSession::IsSupported()
// may throw under certain environments, such as when running
// Qt in a headless Windows environment.
try {
return GraphicsCaptureSession::IsSupported();
} catch (const winrt::hresult_error &err) {
qCWarning(qLcWindowCaptureUwp)
<< "GraphicsCaptureSession::IsSupported() threw exception:"
<< QString::fromWCharArray(err.message().c_str());
return false;
} catch (const std::exception &err) {
qCWarning(qLcWindowCaptureUwp)
<< "GraphicsCaptureSession::IsSupported() failed, threw exception:"
<< err.what();
return false;
}
}
QVideoFrameFormat QFFmpegWindowCaptureUwp::frameFormat() const
{
if (m_grabber)
return m_grabber->frameFormat();
return {};
}
QT_END_NAMESPACE