Revision control
Copy as Markdown
// Copyright (C) 2026 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpeghwaccel_cuda_p.h"
#include <QtFFmpegMediaPluginImpl/private/qffmpeg_p.h>
#include <QtMultimedia/private/qhwvideobuffer_p.h>
#include <QtMultimedia/private/qmultimedia_ranges_p.h>
#include <QtMultimedia/private/qmultimediautils_p.h>
#include <QtGui/rhi/qrhi.h>
#include <QtGui/rhi/qrhi_platform.h>
#include <QtCore/qlibrary.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qscopeguard.h>
#include <QtCore/quuid.h>
#include <QtCore/private/quniquehandle_types_p.h>
#include <array>
#include <optional>
#if QT_CONFIG(opengl)
# include <QtGui/qopenglcontext.h>
# include <QtGui/qopengl.h>
# include <QtGui/qopenglfunctions.h>
# ifndef GL_RG16F
# define GL_RG16F 0x822F
# endif
# ifndef GL_RG16
# define GL_RG16 0x822C
# endif
# ifndef GL_R16
# define GL_R16 0x822A
# endif
#endif
#if QT_CONFIG(vulkan)
# include <QtMultimedia/private/qmultimedia_vulkan_support_p.h>
# include <QtGui/qvulkanfunctions.h>
# include <QtGui/qvulkaninstance.h>
# include <vulkan/vulkan.h>
#endif
#if QT_CONFIG(opengl) || QT_CONFIG(vulkan)
# define QT_FFMPEG_HAS_CUDA_TEXTURE_INTEROP 1
#endif
#if QT_CONFIG(opengl) && (!QT_CONFIG(opengles2) || QT_CONFIG(opengles3))
# define QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP 1
#endif
extern "C" {
// dynlink_cuda.h declares the CUDA driver API without requiring a CUDA Toolkit install;
// including it first also makes libavutil/hwcontext_cuda.h skip its own "<cuda.h>" fallback.
// The header is vendored in src/3rdparty/nv-codec-headers, so it is always available.
#include <ffnvcodec/dynlink_cuda.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_cuda.h>
}
QT_BEGIN_NAMESPACE
using namespace Qt::StringLiterals;
Q_STATIC_LOGGING_CATEGORY(qLHWAccelCUDA, "qt.multimedia.ffmpeg.hwaccelcuda")
namespace QFFmpeg {
template <typename T>
using PerPlaneArray = std::array<T, QVideoTextureHelper::TextureDescription::maxPlanes>;
#if QT_CONFIG(opengl) || QT_CONFIG(vulkan)
namespace {
class CudaDriverApi
{
public:
static const CudaDriverApi *get()
{
static std::optional<CudaDriverApi> singleton = []() -> std::optional<CudaDriverApi> {
CudaDriverApi api;
if (!api.isValid())
return std::nullopt;
return std::optional<CudaDriverApi>{ std::move(api) };
}();
return singleton ? &*singleton : nullptr;
}
bool pushCurrent(CUcontext ctx) const { return cuCtxPushCurrent_v2(ctx) == CUDA_SUCCESS; }
bool popCurrent() const
{
CUcontext popped = nullptr;
return cuCtxPopCurrent_v2(&popped) == CUDA_SUCCESS;
}
tcuGraphicsGLRegisterImage *cuGraphicsGLRegisterImage = nullptr;
tcuGraphicsUnregisterResource *cuGraphicsUnregisterResource = nullptr;
tcuGraphicsMapResources *cuGraphicsMapResources = nullptr;
tcuGraphicsUnmapResources *cuGraphicsUnmapResources = nullptr;
tcuGraphicsSubResourceGetMappedArray *cuGraphicsSubResourceGetMappedArray = nullptr;
tcuMemcpy2D_v2 *cuMemcpy2D_v2 = nullptr;
tcuCtxPushCurrent_v2 *cuCtxPushCurrent_v2 = nullptr;
tcuCtxPopCurrent_v2 *cuCtxPopCurrent_v2 = nullptr;
tcuCtxGetDevice *cuCtxGetDevice = nullptr;
tcuStreamSynchronize *cuStreamSynchronize = nullptr;
tcuDeviceGetUuid *cuDeviceGetUuid = nullptr;
tcuImportExternalMemory *cuImportExternalMemory = nullptr;
tcuDestroyExternalMemory *cuDestroyExternalMemory = nullptr;
tcuExternalMemoryGetMappedMipmappedArray *cuExternalMemoryGetMappedMipmappedArray = nullptr;
tcuMipmappedArrayGetLevel *cuMipmappedArrayGetLevel = nullptr;
tcuGetErrorString *cuGetErrorString = nullptr;
private:
CudaDriverApi()
: m_library{
std::make_unique<QLibrary>(u"cuda"_s),
}
{
if (!m_library->load()) {
qCDebug(qLHWAccelCUDA)
<< "libcuda.so.1 could not be loaded:" << m_library->errorString();
m_library = nullptr;
return;
}
# define RESOLVE(name) \
name = reinterpret_cast<decltype(name)>(m_library->resolve(#name)); \
if (!name) { \
qCDebug(qLHWAccelCUDA) << "libcuda.so.1 is missing symbol:" << #name; \
m_library = nullptr; \
return; \
}
RESOLVE(cuGraphicsGLRegisterImage)
RESOLVE(cuGraphicsUnregisterResource)
RESOLVE(cuGraphicsMapResources)
RESOLVE(cuGraphicsUnmapResources)
RESOLVE(cuGraphicsSubResourceGetMappedArray)
RESOLVE(cuMemcpy2D_v2)
RESOLVE(cuCtxPushCurrent_v2)
RESOLVE(cuCtxPopCurrent_v2)
RESOLVE(cuCtxGetDevice)
RESOLVE(cuStreamSynchronize)
RESOLVE(cuDeviceGetUuid)
RESOLVE(cuImportExternalMemory)
RESOLVE(cuDestroyExternalMemory)
RESOLVE(cuExternalMemoryGetMappedMipmappedArray)
RESOLVE(cuMipmappedArrayGetLevel)
RESOLVE(cuGetErrorString)
# undef RESOLVE
}
bool isValid() const { return bool(m_library); }
std::unique_ptr<QLibrary> m_library;
};
class CudaContextGuard
{
public:
static std::optional<CudaContextGuard> create(const CudaDriverApi &api, CUcontext ctx)
{
if (!api.pushCurrent(ctx))
return std::nullopt;
return CudaContextGuard(api);
}
CudaContextGuard(CudaContextGuard &&other) noexcept : m_api(other.m_api)
{
other.m_api = nullptr;
}
CudaContextGuard &operator=(CudaContextGuard &&) = delete;
~CudaContextGuard()
{
if (m_api)
m_api->popCurrent();
}
Q_DISABLE_COPY(CudaContextGuard)
private:
explicit CudaContextGuard(const CudaDriverApi &api) : m_api(&api) { }
const CudaDriverApi *m_api;
};
class CudaResult
{
public:
CudaResult(CUresult result, const CudaDriverApi &api) : m_result(result), m_api(api) { }
private:
friend QDebug operator<<(QDebug debug, const CudaResult &r)
{
const QDebugStateSaver saver(debug);
const char *errorString = nullptr;
r.m_api.cuGetErrorString(r.m_result, &errorString);
debug.nospace().noquote() << "CUresult(0x" << Qt::hex << qToUnderlying(r.m_result)
<< Qt::dec << ", "
<< (errorString ? errorString : "unknown error") << ')';
return debug;
}
CUresult m_result;
const CudaDriverApi &m_api;
};
// The layout of one plane of a CUDA-decoded pixel format, backend-agnostic: just enough to
// compute plane sizes and the byte width of the CUDA memcpy.
struct CudaPlaneGeometry
{
int sampleSize = 1; // bytes per sample: 1 for 8-bit formats, 2 for 10/16-bit-in-16-bit
int numChannels = 1; // 1 (luma) or 2 (interleaved chroma)
// The plane's size is the frame size divided by these factors, e.g. 2/2 for a 4:2:0
// chroma plane. Default to 1, not 0, since these are divisors.
int widthDivisor = 1;
int heightDivisor = 1;
int bytesPerTexel() const { return sampleSize * numChannels; }
};
struct CudaFrameGeometry
{
int planeCount = 0;
PerPlaneArray<CudaPlaneGeometry> planes{};
};
// NVDEC only ever decodes into semi-planar (2-plane, chroma-subsampled) formats - never
// fully-planar ones - so this table only needs to cover that family.
std::optional<CudaFrameGeometry> cudaFrameGeometry(AVPixelFormat swFormat)
{
switch (swFormat) {
case AV_PIX_FMT_NV12:
case AV_PIX_FMT_NV21:
return CudaFrameGeometry{
2,
{ {
CudaPlaneGeometry{ 1, 1, 1, 1 },
CudaPlaneGeometry{ 1, 2, 2, 2 },
} },
};
case AV_PIX_FMT_P010:
case AV_PIX_FMT_P016:
return CudaFrameGeometry{
2,
{ {
CudaPlaneGeometry{ 2, 1, 1, 1 },
CudaPlaneGeometry{ 2, 2, 2, 2 },
} },
};
default:
return std::nullopt;
}
}
// Exposes each plane's texture as a native handle: a GLuint on OpenGL, a VkImage on Vulkan.
class CudaTextureHandles final : public QVideoFrameTexturesHandles
{
public:
CudaTextureHandles(int nPlanes, PerPlaneArray<quint64> handles,
std::shared_ptr<void> parentKeepAlive)
: m_nPlanes(nPlanes), m_handles(handles), m_parentKeepAlive(std::move(parentKeepAlive))
{
}
quint64 textureHandle(QRhi &, int plane) override
{
return plane < m_nPlanes ? m_handles[plane] : 0;
}
private:
const int m_nPlanes;
const PerPlaneArray<quint64> m_handles;
const std::shared_ptr<void>
m_parentKeepAlive; // keeps the converter (and its GL/Vulkan resources) alive
};
// ---------------------------------------------------------------------------------------------
// OpenGL backend
// ---------------------------------------------------------------------------------------------
# if QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP
struct GlPlaneFormat
{
GLenum internalFormat;
GLenum format;
GLenum type;
};
GlPlaneFormat glPlaneFormat(const CudaPlaneGeometry &geometry)
{
const GLenum type = geometry.sampleSize == 1 ? GL_UNSIGNED_BYTE : GL_UNSIGNED_SHORT;
if (geometry.numChannels == 1)
return GlPlaneFormat{
GLenum(geometry.sampleSize == 1 ? GL_R8 : GL_R16),
GLenum(GL_RED),
type,
};
return GlPlaneFormat{
GLenum(geometry.sampleSize == 1 ? GL_RG8 : GL_RG16),
GLenum(GL_RG),
type,
};
}
struct GlPlaneResource
{
~GlPlaneResource() { Q_ASSERT(texture == 0 && !cudaResource); }
void release(const CudaDriverApi *cudaApi, QOpenGLFunctions &functions)
{
if (cudaResource && cudaApi)
cudaApi->cuGraphicsUnregisterResource(cudaResource);
if (texture)
functions.glDeleteTextures(1, &texture);
cudaResource = nullptr;
texture = 0;
}
GLuint texture = 0;
CUgraphicsResource cudaResource = nullptr;
};
class CUDAGLTextureConverter final : public TextureConverterBackend
{
public:
static std::shared_ptr<TextureConverterBackend> create(QRhi *rhi)
{
if (!rhi)
return nullptr;
auto *api = CudaDriverApi::get();
if (!api) {
qCDebug(qLHWAccelCUDA) << "CUDA driver API is not available, disabling";
return nullptr;
}
return std::make_shared<CUDAGLTextureConverter>(rhi, api);
}
explicit CUDAGLTextureConverter(QRhi *rhi, const CudaDriverApi *api)
: TextureConverterBackend(rhi), m_api{ api }
{
}
~CUDAGLTextureConverter() override { releasePlaneResources(); }
QVideoFrameTexturesHandlesUPtr
createTextureHandles(AVFrame *frame, QVideoFrameTexturesHandlesUPtr /*oldHandles*/) override
{
if (frame->format != AV_PIX_FMT_CUDA || !frame->hw_frames_ctx)
return nullptr;
auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
const std::optional<CudaFrameGeometry> geometry = cudaFrameGeometry(framesCtx->sw_format);
if (!geometry) {
qWarning() << "CUDATextureConverter does not support sw_format" << framesCtx->sw_format;
return nullptr;
}
auto *deviceCtx = avFrameDeviceContext(frame);
if (!deviceCtx)
return nullptr;
auto *cudaCtx = reinterpret_cast<AVCUDADeviceContext *>(deviceCtx->hwctx);
std::optional<CudaContextGuard> guard = CudaContextGuard::create(*m_api, cudaCtx->cuda_ctx);
if (!guard) {
qWarning() << "Failed to push the CUDA context current; disabling";
rhi = nullptr;
return nullptr;
}
m_cudaContext = cudaCtx->cuda_ctx;
if (!ensurePlaneResources(frame))
return nullptr;
PerPlaneArray<CUgraphicsResource> resources{};
for (int i = 0; i < geometry->planeCount; ++i)
resources[i] = m_planes[i]->cudaResource;
const CUresult mapResult =
m_api->cuGraphicsMapResources(geometry->planeCount, resources.data(), nullptr);
if (mapResult != CUDA_SUCCESS) {
qWarning() << "cuGraphicsMapResources failed:" << CudaResult(mapResult, *m_api);
return nullptr;
}
auto unmapGuard = qScopeGuard([&] {
m_api->cuGraphicsUnmapResources(geometry->planeCount, resources.data(), nullptr);
});
PerPlaneArray<quint64> handles{};
for (int i = 0; i < geometry->planeCount; ++i) {
const CudaPlaneGeometry &planeGeometry = geometry->planes[i];
const QSize planeSize(frame->width / planeGeometry.widthDivisor,
frame->height / planeGeometry.heightDivisor);
CUarray array = nullptr;
const CUresult subResourceResult =
m_api->cuGraphicsSubResourceGetMappedArray(&array, resources[i], 0, 0);
if (subResourceResult != CUDA_SUCCESS) {
qWarning() << "cuGraphicsSubResourceGetMappedArray failed for plane" << i << ":"
<< CudaResult(subResourceResult, *m_api);
return nullptr;
}
const CUDA_MEMCPY2D copy{
.srcXInBytes = 0,
.srcY = 0,
.srcMemoryType = CU_MEMORYTYPE_DEVICE,
.srcHost = nullptr,
.srcDevice = reinterpret_cast<CUdeviceptr>(frame->data[i]),
.srcArray = nullptr,
.srcPitch = size_t(frame->linesize[i]),
.dstXInBytes = 0,
.dstY = 0,
.dstMemoryType = CU_MEMORYTYPE_ARRAY,
.dstHost = nullptr,
.dstDevice = 0,
.dstArray = array,
.dstPitch = 0,
.WidthInBytes = size_t(planeSize.width()) * size_t(planeGeometry.bytesPerTexel()),
.Height = size_t(planeSize.height()),
};
const CUresult copyResult = m_api->cuMemcpy2D_v2(©);
if (copyResult != CUDA_SUCCESS) {
qWarning() << "cuMemcpy2D_v2 failed for plane" << i << ":"
<< CudaResult(copyResult, *m_api);
return nullptr;
}
handles[i] = quint64(m_planes[i]->texture);
}
return std::make_unique<CudaTextureHandles>(geometry->planeCount, handles,
shared_from_this());
}
private:
bool ensurePlaneResources(AVFrame *frame)
{
auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
const std::optional<CudaFrameGeometry> maybeGeometry =
cudaFrameGeometry(framesCtx->sw_format);
Q_ASSERT(maybeGeometry); // already validated by createTextureHandles
const CudaFrameGeometry &geometry = *maybeGeometry;
const QSize frameSize(frame->width, frame->height);
const bool canReuse = frameSize == m_planeResourcesSize
&& framesCtx->sw_format == m_planeResourcesFormat
&& ranges::all_of(QSpan(m_planes.data(), geometry.planeCount),
[](const auto &plane) {
return bool(plane);
});
if (canReuse)
return true;
releasePlaneResources();
m_planeResourcesSize = frameSize;
m_planeResourcesFormat = framesCtx->sw_format;
auto *glContext =
static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles())->context;
QOpenGLFunctions functions(glContext);
for (int i = 0; i < geometry.planeCount; ++i) {
const CudaPlaneGeometry &planeGeometry = geometry.planes[i];
const QSize planeSize(m_planeResourcesSize.width() / planeGeometry.widthDivisor,
m_planeResourcesSize.height() / planeGeometry.heightDivisor);
const GlPlaneFormat format = glPlaneFormat(planeGeometry);
auto plane = std::make_unique<GlPlaneResource>();
functions.glGenTextures(1, &plane->texture);
functions.glBindTexture(GL_TEXTURE_2D, plane->texture);
functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
functions.glTexImage2D(GL_TEXTURE_2D, 0, format.internalFormat, planeSize.width(),
planeSize.height(), 0, format.format, format.type, nullptr);
functions.glBindTexture(GL_TEXTURE_2D, 0);
const CUresult result = m_api->cuGraphicsGLRegisterImage(
&plane->cudaResource, plane->texture, GL_TEXTURE_2D,
CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD);
if (result != CUDA_SUCCESS) {
qWarning() << "cuGraphicsGLRegisterImage failed for plane" << i << ":"
<< CudaResult(result, *m_api);
plane->release(m_api, functions);
return false;
}
m_planes[i] = std::move(plane);
}
return true;
}
void releasePlaneResources()
{
if (!rhi)
return;
std::optional<CudaContextGuard> guard = m_cudaContext && m_api
? CudaContextGuard::create(*m_api, m_cudaContext)
: std::nullopt;
rhi->makeThreadLocalNativeContextCurrent();
auto *glContext =
static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles())->context;
QOpenGLFunctions functions(glContext);
for (auto &plane : m_planes) {
if (!plane)
continue;
plane->release(guard ? m_api : nullptr, functions);
plane.reset();
}
}
const CudaDriverApi *const m_api;
PerPlaneArray<std::unique_ptr<GlPlaneResource>> m_planes;
QSize m_planeResourcesSize;
AVPixelFormat m_planeResourcesFormat = AV_PIX_FMT_NONE;
CUcontext m_cudaContext = nullptr;
};
# endif // QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP
// ---------------------------------------------------------------------------------------------
// Vulkan backend
// ---------------------------------------------------------------------------------------------
# if QT_CONFIG(vulkan)
// Pass a device to resolve a device-level function (e.g. of VK_KHR_external_memory_fd), which
// yields nullptr if the extension is not enabled on that device. Without a device, an
// instance-level function is resolved.
template <typename Func>
Func resolveVulkanFunctionOrWarn(QVulkanInstance &inst, const char *name,
VkDevice dev = VK_NULL_HANDLE)
{
using namespace QtMultimediaPrivate;
Func func = dev ? resolveVulkanDeviceFunction<Func>(inst, dev, name)
: resolveVulkanFunction<Func>(inst, name);
if (!func)
qCDebug(qLHWAccelCUDA) << "Failed to resolve Vulkan function" << name;
return func;
}
// vkGetPhysicalDeviceProperties2 is core since Vulkan 1.1, with vkGetPhysicalDeviceProperties2KHR
// as the (functionally identical) extension entry point on older API versions/instances.
PFN_vkGetPhysicalDeviceProperties2 resolveVkGetPhysicalDeviceProperties2(QVulkanInstance &inst)
{
using namespace QtMultimediaPrivate;
PFN_vkGetPhysicalDeviceProperties2 func =
resolveVulkanFunction<PFN_vkGetPhysicalDeviceProperties2>(
inst, "vkGetPhysicalDeviceProperties2");
if (!func) {
func = resolveVulkanFunctionOrWarn<PFN_vkGetPhysicalDeviceProperties2>(
inst, "vkGetPhysicalDeviceProperties2KHR");
}
return func;
}
QUuid uuidFromCUuuid(const CUuuid &cuUuid)
{
return QUuid::fromRfc4122(QByteArrayView(reinterpret_cast<const char *>(cuUuid.bytes), 16));
}
// Resolves the Vulkan state/functions needed to export a plane's VkImage memory as a POSIX
// fd (VK_KHR_external_memory_fd) for import into CUDA. Bound to one QRhi::Vulkan instance.
class CudaVulkanInterop
{
public:
explicit CudaVulkanInterop(const QRhiVulkanNativeHandles &handles)
: inst(handles.inst),
physDev(handles.physDev),
dev(handles.dev),
deviceFuncs(handles.inst->deviceFunctions(handles.dev)),
instFuncs(handles.inst->functions()),
vkGetMemoryFdKHR(resolveVulkanFunctionOrWarn<PFN_vkGetMemoryFdKHR>(
*handles.inst, "vkGetMemoryFdKHR", handles.dev)),
vkGetPhysicalDeviceProperties2(resolveVkGetPhysicalDeviceProperties2(*handles.inst))
{
}
static std::unique_ptr<CudaVulkanInterop> create(QRhi *rhi)
{
Q_ASSERT(rhi->backend() == QRhi::Vulkan);
const auto *handles = static_cast<const QRhiVulkanNativeHandles *>(rhi->nativeHandles());
if (!handles || !handles->inst || !handles->dev || !handles->physDev)
return nullptr;
auto interop = std::make_unique<CudaVulkanInterop>(*handles);
if (!interop->deviceFuncs || !interop->instFuncs)
return nullptr;
if (!interop->vkGetMemoryFdKHR || !interop->vkGetPhysicalDeviceProperties2) {
qCDebug(qLHWAccelCUDA)
<< "VK_KHR_external_memory_fd or vkGetPhysicalDeviceProperties2 is either "
"unavailable or not enabled on this QRhi's VkDevice/VkInstance, disabling. "
"To use the CUDA texture interop with Vulkan, enable the device extensions "
"VK_KHR_external_memory and VK_KHR_external_memory_fd when creating the "
"QRhi, e.g. with QQuickGraphicsConfiguration::setDeviceExtensions() or "
"QRhiVulkanInitParams::deviceExtensions";
return nullptr;
}
return interop;
}
enum class MatchResult : uint8_t {
SameGpu,
DifferentGpu,
Unknown, // Vulkan device UUID could not be queried
};
MatchResult matchesCudaDevice(const CUuuid &cudaUuid) const
{
using namespace QtMultimediaPrivate;
VkPhysicalDeviceIDProperties idProps{
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES,
.pNext = nullptr,
.deviceUUID = {},
.driverUUID = {},
.deviceLUID = {},
.deviceNodeMask = 0,
.deviceLUIDValid = VK_FALSE,
};
VkPhysicalDeviceProperties2 props2{
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
.pNext = &idProps,
.properties = {},
};
vkGetPhysicalDeviceProperties2(physDev, &props2);
const QUuid vkDeviceUuid = uuidFromVkUuid(idProps.deviceUUID);
if (vkDeviceUuid.isNull())
return MatchResult::Unknown;
const QUuid cuDeviceUuid = uuidFromCUuuid(cudaUuid);
return (vkDeviceUuid == cuDeviceUuid) ? MatchResult::SameGpu : MatchResult::DifferentGpu;
}
std::optional<uint32_t> findDeviceLocalMemoryTypeIndex(uint32_t memoryTypeBits) const
{
VkPhysicalDeviceMemoryProperties memProps{};
instFuncs->vkGetPhysicalDeviceMemoryProperties(physDev, &memProps);
for (uint32_t i = 0; i < memProps.memoryTypeCount; ++i) {
const bool allowed = memoryTypeBits & (1u << i);
const bool deviceLocal =
memProps.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
if (allowed && deviceLocal)
return i;
}
return std::nullopt;
}
QVulkanInstance *const inst;
const VkPhysicalDevice physDev;
const VkDevice dev;
QVulkanDeviceFunctions *const deviceFuncs;
QVulkanFunctions *const instFuncs;
const PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR;
const PFN_vkGetPhysicalDeviceProperties2 vkGetPhysicalDeviceProperties2;
};
struct VulkanPlaneFormat
{
VkFormat vkFormat;
CUarray_format cuArrayFormat;
};
VulkanPlaneFormat vulkanPlaneFormat(const CudaPlaneGeometry &geometry)
{
const CUarray_format cuFormat =
geometry.sampleSize == 1 ? CU_AD_FORMAT_UNSIGNED_INT8 : CU_AD_FORMAT_UNSIGNED_INT16;
if (geometry.numChannels == 1)
return VulkanPlaneFormat{
geometry.sampleSize == 1 ? VK_FORMAT_R8_UNORM : VK_FORMAT_R16_UNORM,
cuFormat,
};
return VulkanPlaneFormat{
geometry.sampleSize == 1 ? VK_FORMAT_R8G8_UNORM : VK_FORMAT_R16G16_UNORM,
cuFormat,
};
}
// Vulkan interop (VK_KHR_external_memory_fd + cuImportExternalMemory).
struct VulkanPlaneResource
{
~VulkanPlaneResource()
{
Q_ASSERT(image == VK_NULL_HANDLE && memory == VK_NULL_HANDLE && !externalMemory);
}
void release(const CudaDriverApi *cudaApi, CudaVulkanInterop &vulkan)
{
if (externalMemory && cudaApi)
cudaApi->cuDestroyExternalMemory(externalMemory);
if (memory != VK_NULL_HANDLE)
vulkan.deviceFuncs->vkFreeMemory(vulkan.dev, memory, nullptr);
if (image != VK_NULL_HANDLE)
vulkan.deviceFuncs->vkDestroyImage(vulkan.dev, image, nullptr);
externalMemory = nullptr;
mipmappedArray = nullptr;
array = nullptr;
memory = VK_NULL_HANDLE;
image = VK_NULL_HANDLE;
}
VkImage image = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
CUexternalMemory externalMemory = nullptr;
CUmipmappedArray mipmappedArray = nullptr;
CUarray array = nullptr; // level-0 view of mipmappedArray, cached for the copy
};
class CUDAVulkanTextureConverter final : public TextureConverterBackend
{
using MatchResult = CudaVulkanInterop::MatchResult;
public:
static std::shared_ptr<TextureConverterBackend> create(QRhi *rhi)
{
if (!rhi)
return nullptr;
auto *api = CudaDriverApi::get();
if (!api) {
qCDebug(qLHWAccelCUDA) << "CUDA driver API is not available, disabling";
return nullptr;
}
auto vulkanApi = CudaVulkanInterop::create(rhi);
if (!vulkanApi) {
qCDebug(qLHWAccelCUDA) << "CUDA-Vulkan interop is not available, disabling";
return nullptr;
}
return std::make_shared<CUDAVulkanTextureConverter>(rhi, api, std::move(vulkanApi));
}
explicit CUDAVulkanTextureConverter(QRhi *rhi, const CudaDriverApi *api,
std::unique_ptr<CudaVulkanInterop> vulkanApi)
: TextureConverterBackend(rhi), m_api{ api }, m_vulkan{ std::move(vulkanApi) }
{
}
~CUDAVulkanTextureConverter() override { releasePlaneResources(); }
QVideoFrameTexturesHandlesUPtr
createTextureHandles(AVFrame *frame, QVideoFrameTexturesHandlesUPtr /*oldHandles*/) override
{
if (frame->format != AV_PIX_FMT_CUDA || !frame->hw_frames_ctx)
return nullptr;
auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
const std::optional<CudaFrameGeometry> geometry = cudaFrameGeometry(framesCtx->sw_format);
if (!geometry) {
qWarning() << "CUDATextureConverter does not support sw_format" << framesCtx->sw_format;
return nullptr;
}
auto *deviceCtx = avFrameDeviceContext(frame);
if (!deviceCtx)
return nullptr;
auto *cudaCtx = reinterpret_cast<AVCUDADeviceContext *>(deviceCtx->hwctx);
std::optional<CudaContextGuard> guard = CudaContextGuard::create(*m_api, cudaCtx->cuda_ctx);
if (!guard) {
qWarning() << "Failed to push the CUDA context current; disabling";
rhi = nullptr;
return nullptr;
}
m_cudaContext = cudaCtx->cuda_ctx;
if (!ensurePlaneResources(frame))
return nullptr;
PerPlaneArray<quint64> handles{};
for (int i = 0; i < geometry->planeCount; ++i) {
const CudaPlaneGeometry &planeGeometry = geometry->planes[i];
const QSize planeSize(frame->width / planeGeometry.widthDivisor,
frame->height / planeGeometry.heightDivisor);
const CUDA_MEMCPY2D copy{
.srcXInBytes = 0,
.srcY = 0,
.srcMemoryType = CU_MEMORYTYPE_DEVICE,
.srcHost = nullptr,
.srcDevice = reinterpret_cast<CUdeviceptr>(frame->data[i]),
.srcArray = nullptr,
.srcPitch = size_t(frame->linesize[i]),
.dstXInBytes = 0,
.dstY = 0,
.dstMemoryType = CU_MEMORYTYPE_ARRAY,
.dstHost = nullptr,
.dstDevice = 0,
.dstArray = m_planes[i]->array,
.dstPitch = 0,
.WidthInBytes = size_t(planeSize.width()) * size_t(planeGeometry.bytesPerTexel()),
.Height = size_t(planeSize.height()),
};
const CUresult copyResult = m_api->cuMemcpy2D_v2(©);
if (copyResult != CUDA_SUCCESS) {
qWarning() << "cuMemcpy2D_v2 failed for plane" << i << ":"
<< CudaResult(copyResult, *m_api);
return nullptr;
}
handles[i] = quint64(m_planes[i]->image);
}
// No external-semaphore pipelining (see the class docstring): block until the
// copies actually land before handing the VkImages to the RHI. cuMemcpy2D_v2
// runs on the default stream, so synchronizing that stream is sufficient.
const CUresult syncResult = m_api->cuStreamSynchronize(nullptr);
if (syncResult != CUDA_SUCCESS) {
qWarning() << "cuStreamSynchronize failed:" << CudaResult(syncResult, *m_api);
return nullptr;
}
return std::make_unique<CudaTextureHandles>(geometry->planeCount, handles,
shared_from_this());
}
private:
bool ensurePlaneResources(AVFrame *frame)
{
auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
const std::optional<CudaFrameGeometry> maybeGeometry =
cudaFrameGeometry(framesCtx->sw_format);
Q_ASSERT(maybeGeometry); // already validated by createTextureHandles
const CudaFrameGeometry &geometry = *maybeGeometry;
const QSize frameSize(frame->width, frame->height);
const bool canReuse = frameSize == m_planeResourcesSize
&& framesCtx->sw_format == m_planeResourcesFormat
&& ranges::all_of(QSpan(m_planes.data(), geometry.planeCount),
[](const auto &plane) {
return bool(plane);
});
if (canReuse)
return true;
releasePlaneResources();
m_planeResourcesSize = frameSize;
m_planeResourcesFormat = framesCtx->sw_format;
if (m_vulkanDeviceMatches == MatchResult::Unknown) {
CUdevice device = 0;
CUuuid cudaUuid{};
const bool ok = m_api->cuCtxGetDevice(&device) == CUDA_SUCCESS
&& m_api->cuDeviceGetUuid(&cudaUuid, device) == CUDA_SUCCESS;
m_vulkanDeviceMatches =
ok ? m_vulkan->matchesCudaDevice(cudaUuid) : MatchResult::Unknown;
if (m_vulkanDeviceMatches == MatchResult::Unknown) {
qWarning() << "CUDATextureConverter: could not determine the QRhi's Vulkan "
"physical device UUID (missing "
"VK_KHR_get_physical_device_properties2 on the VkInstance?); "
"disabling";
} else if (m_vulkanDeviceMatches == MatchResult::DifferentGpu) {
qWarning() << "CUDATextureConverter: the CUDA device does not match the QRhi's "
"Vulkan physical device (e.g. a multi-GPU system); disabling";
}
}
if (m_vulkanDeviceMatches != MatchResult::SameGpu)
return false;
for (int i = 0; i < geometry.planeCount; ++i) {
const CudaPlaneGeometry &planeGeometry = geometry.planes[i];
const QSize planeSize(m_planeResourcesSize.width() / planeGeometry.widthDivisor,
m_planeResourcesSize.height() / planeGeometry.heightDivisor);
const VulkanPlaneFormat format = vulkanPlaneFormat(planeGeometry);
auto plane = std::make_unique<VulkanPlaneResource>();
auto releaseGuard = qScopeGuard([&] {
plane->release(m_api, *m_vulkan);
});
const VkExternalMemoryImageCreateInfo extImageInfo{
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
.pNext = nullptr,
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
};
// TRANSFER_SRC is not needed by the RHI (it only samples this texture), but it
// costs nothing to declare and lets tests validate the copy via a readback.
const VkImageCreateInfo imageInfo{
.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
.pNext = &extImageInfo,
.flags = 0,
.imageType = VK_IMAGE_TYPE_2D,
.format = format.vkFormat,
.extent =
VkExtent3D{
.width = uint32_t(planeSize.width()),
.height = uint32_t(planeSize.height()),
.depth = 1u,
},
.mipLevels = 1,
.arrayLayers = 1,
.samples = VK_SAMPLE_COUNT_1_BIT,
.tiling = VK_IMAGE_TILING_OPTIMAL,
.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT
| VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = nullptr,
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
};
if (m_vulkan->deviceFuncs->vkCreateImage(m_vulkan->dev, &imageInfo, nullptr,
&plane->image)
!= VK_SUCCESS) {
qWarning() << "CUDATextureConverter: vkCreateImage failed for plane" << i;
return false;
}
VkMemoryRequirements memReq{};
m_vulkan->deviceFuncs->vkGetImageMemoryRequirements(m_vulkan->dev, plane->image,
&memReq);
const std::optional<uint32_t> memoryTypeIndex =
m_vulkan->findDeviceLocalMemoryTypeIndex(memReq.memoryTypeBits);
if (!memoryTypeIndex) {
qWarning() << "CUDATextureConverter: no suitable Vulkan memory type for plane" << i;
return false;
}
const VkExportMemoryAllocateInfo exportInfo{
.sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,
.pNext = nullptr,
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
};
const VkMemoryAllocateInfo allocInfo{
.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
.pNext = &exportInfo,
.allocationSize = memReq.size,
.memoryTypeIndex = *memoryTypeIndex,
};
if (m_vulkan->deviceFuncs->vkAllocateMemory(m_vulkan->dev, &allocInfo, nullptr,
&plane->memory)
!= VK_SUCCESS) {
qWarning() << "CUDATextureConverter: vkAllocateMemory failed for plane" << i;
return false;
}
if (m_vulkan->deviceFuncs->vkBindImageMemory(m_vulkan->dev, plane->image, plane->memory,
0)
!= VK_SUCCESS) {
qWarning() << "CUDATextureConverter: vkBindImageMemory failed for plane" << i;
return false;
}
const VkMemoryGetFdInfoKHR getFdInfo{
.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
.pNext = nullptr,
.memory = plane->memory,
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
};
QUniqueFileDescriptorHandle fd;
if (m_vulkan->vkGetMemoryFdKHR(m_vulkan->dev, &getFdInfo, &fd) != VK_SUCCESS) {
qWarning() << "CUDATextureConverter: vkGetMemoryFdKHR failed for plane" << i;
return false;
}
const CUDA_EXTERNAL_MEMORY_HANDLE_DESC extMemDesc{
.type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
.handle = { .fd = fd.get() },
.size = memReq.size,
.flags = 0,
.reserved = {},
};
const CUresult importResult =
m_api->cuImportExternalMemory(&plane->externalMemory, &extMemDesc);
if (importResult != CUDA_SUCCESS) {
qWarning() << "CUDATextureConverter: cuImportExternalMemory failed for plane" << i
<< ":" << CudaResult(importResult, *m_api);
return false;
}
std::ignore = fd.release(); // import succeeded, ownership transferred to CUDA
const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC arrayDesc{
.offset = 0,
.arrayDesc =
CUDA_ARRAY3D_DESCRIPTOR{
.Width = size_t(planeSize.width()),
.Height = size_t(planeSize.height()),
.Depth = 0,
.Format = format.cuArrayFormat,
.NumChannels = unsigned(planeGeometry.numChannels),
.Flags = 0,
},
.numLevels = 1,
.reserved = {},
};
const CUresult mappedArrayResult = m_api->cuExternalMemoryGetMappedMipmappedArray(
&plane->mipmappedArray, plane->externalMemory, &arrayDesc);
if (mappedArrayResult != CUDA_SUCCESS) {
qWarning() << "CUDATextureConverter: cuExternalMemoryGetMappedMipmappedArray "
"failed for plane"
<< i << ":" << CudaResult(mappedArrayResult, *m_api);
return false;
}
const CUresult arrayLevelResult =
m_api->cuMipmappedArrayGetLevel(&plane->array, plane->mipmappedArray, 0);
if (arrayLevelResult != CUDA_SUCCESS) {
qWarning() << "CUDATextureConverter: cuMipmappedArrayGetLevel failed for plane" << i
<< ":" << CudaResult(arrayLevelResult, *m_api);
return false;
}
releaseGuard.dismiss();
m_planes[i] = std::move(plane);
}
return true;
}
void releasePlaneResources()
{
if (!rhi)
return;
std::optional<CudaContextGuard> guard = m_cudaContext && m_api
? CudaContextGuard::create(*m_api, m_cudaContext)
: std::nullopt;
for (auto &plane : m_planes) {
if (!plane)
continue;
plane->release(guard ? m_api : nullptr, *m_vulkan);
plane.reset();
}
}
const CudaDriverApi *const m_api;
const std::unique_ptr<CudaVulkanInterop> m_vulkan;
MatchResult m_vulkanDeviceMatches = MatchResult::Unknown;
PerPlaneArray<std::unique_ptr<VulkanPlaneResource>> m_planes;
QSize m_planeResourcesSize;
AVPixelFormat m_planeResourcesFormat = AV_PIX_FMT_NONE;
CUcontext m_cudaContext = nullptr;
};
# endif // QT_CONFIG(vulkan)
} // namespace
#endif // QT_FFMPEG_HAS_CUDA_TEXTURE_INTEROP
// ---------------------------------------------------------------------------------------------
// Factory
// ---------------------------------------------------------------------------------------------
std::shared_ptr<TextureConverterBackend> CUDATextureConverterFactory::create(QRhi *rhi)
{
if (rhi) {
switch (rhi->backend()) {
#if QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP
case QRhi::OpenGLES2:
return CUDAGLTextureConverter::create(rhi);
#endif
#if QT_CONFIG(vulkan)
case QRhi::Vulkan:
return CUDAVulkanTextureConverter::create(rhi);
#endif
default:
break;
}
}
return nullptr;
}
} // namespace QFFmpeg
QT_END_NAMESPACE