Revision control

Copy as Markdown

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpeghwaccel_d3d11_p.h"
#include <qvideoframeformat.h>
#include <private/qvideotexturehelper_p.h>
#include <QtCore/private/qcomptr_p.h>
#include <private/quniquehandle_p.h>
#include <rhi/qrhi.h>
#include <qopenglfunctions.h>
#include <qdebug.h>
#include <qloggingcategory.h>
#include <libavutil/hwcontext_d3d11va.h>
#include <d3d11_1.h>
#include <dxgi1_2.h>
QT_BEGIN_NAMESPACE
namespace {
Q_LOGGING_CATEGORY(qLcMediaFFmpegHWAccel, "qt.multimedia.hwaccel");
ComPtr<ID3D11Device1> GetD3DDevice(QRhi *rhi)
{
const auto native = static_cast<const QRhiD3D11NativeHandles *>(rhi->nativeHandles());
if (!native)
return {};
const ComPtr<ID3D11Device> rhiDevice = static_cast<ID3D11Device *>(native->dev);
ComPtr<ID3D11Device1> dev1;
if (rhiDevice.As(&dev1) != S_OK)
return nullptr;
return dev1;
}
ComPtr<ID3D11Texture2D> getAvFrameTexture(const AVFrame *frame)
{
return reinterpret_cast<ID3D11Texture2D *>(frame->data[0]);
}
int getAvFramePoolIndex(const AVFrame *frame)
{
return static_cast<int>(reinterpret_cast<intptr_t>(frame->data[1]));
}
const AVD3D11VADeviceContext *getHwDeviceContext(const AVHWDeviceContext *ctx)
{
return static_cast<AVD3D11VADeviceContext *>(ctx->hwctx);
}
AVBufferRef *wrapTextureAsBuffer(const ComPtr<ID3D11Texture2D> &tex)
{
Q_ASSERT(tex);
AVD3D11FrameDescriptor *avFrameDesc =
static_cast<AVD3D11FrameDescriptor *>(av_mallocz(sizeof(AVD3D11FrameDescriptor)));
avFrameDesc->index = 0;
avFrameDesc->texture = tex.Get();
return av_buffer_create(reinterpret_cast<uint8_t *>(avFrameDesc),
sizeof(AVD3D11FrameDescriptor *), [](void *opaque, uint8_t *data) {
static_cast<ID3D11Texture2D *>(opaque)->Release();
av_free(data);
}, tex.Get(), 0);
}
ComPtr<ID3D11Texture2D> copyTexture(const AVD3D11VADeviceContext *hwDevCtx, const AVFrame *src)
{
const int poolIndex = getAvFramePoolIndex(src);
const ComPtr<ID3D11Texture2D> poolTex = getAvFrameTexture(src);
D3D11_TEXTURE2D_DESC texDesc{};
poolTex->GetDesc(&texDesc);
texDesc.ArraySize = 1;
texDesc.MiscFlags = 0;
texDesc.BindFlags = 0;
ComPtr<ID3D11Texture2D> destTex;
if (hwDevCtx->device->CreateTexture2D(&texDesc, nullptr, &destTex) != S_OK) {
qCCritical(qLcMediaFFmpegHWAccel) << "Unable to copy frame from decoder pool";
return {};
}
hwDevCtx->device_context->CopySubresourceRegion(destTex.Get(), 0, 0, 0, 0, poolTex.Get(),
poolIndex, nullptr);
return destTex;
}
} // namespace
namespace QFFmpeg {
bool TextureBridge::copyToSharedTex(ID3D11Device *dev, ID3D11DeviceContext *ctx,
const ComPtr<ID3D11Texture2D> &tex, UINT index,
const QSize &frameSize)
{
if (!ensureSrcTex(dev, tex, frameSize))
return false;
// Flush to ensure that texture is fully updated before we share it.
ctx->Flush();
if (m_srcMutex->AcquireSync(m_srcKey, INFINITE) != S_OK)
return false;
const UINT width = static_cast<UINT>(frameSize.width());
const UINT height = static_cast<UINT>(frameSize.height());
// A crop box is needed because FFmpeg may have created textures
// that are bigger than the frame size to account for the decoder's
// surface alignment requirements.
const D3D11_BOX crop{ 0, 0, 0, width, height, 1 };
ctx->CopySubresourceRegion(m_srcTex.Get(), 0, 0, 0, 0, tex.Get(), index, &crop);
m_srcMutex->ReleaseSync(m_destKey);
return true;
}
ComPtr<ID3D11Texture2D> TextureBridge::copyFromSharedTex(const ComPtr<ID3D11Device1> &dev,
const ComPtr<ID3D11DeviceContext> &ctx)
{
if (!ensureDestTex(dev))
return {};
if (m_destMutex->AcquireSync(m_destKey, INFINITE) != S_OK)
return {};
ctx->CopySubresourceRegion(m_outputTex.Get(), 0, 0, 0, 0, m_destTex.Get(), 0, nullptr);
m_destMutex->ReleaseSync(m_srcKey);
return m_outputTex;
}
bool TextureBridge::ensureDestTex(const ComPtr<ID3D11Device1> &dev)
{
if (m_destDevice.Get() != dev.Get()) {
// Destination device changed. Recreate texture.
m_destTex = nullptr;
m_destDevice = dev;
}
if (m_destTex)
return true;
if (m_destDevice->OpenSharedResource1(m_sharedHandle.get(), IID_PPV_ARGS(&m_destTex)) != S_OK)
return false;
CD3D11_TEXTURE2D_DESC desc{};
m_destTex->GetDesc(&desc);
desc.MiscFlags = 0;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
if (m_destDevice->CreateTexture2D(&desc, nullptr, m_outputTex.ReleaseAndGetAddressOf()) != S_OK)
return false;
if (m_destTex.As(&m_destMutex) != S_OK)
return false;
return true;
}
bool TextureBridge::ensureSrcTex(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex, const QSize &frameSize)
{
if (!isSrcInitialized(dev, tex, frameSize))
return recreateSrc(dev, tex, frameSize);
return true;
}
bool TextureBridge::isSrcInitialized(const ID3D11Device *dev,
const ComPtr<ID3D11Texture2D> &tex,
const QSize &frameSize) const
{
if (!m_srcTex)
return false;
// Check if device has changed
ComPtr<ID3D11Device> texDevice;
m_srcTex->GetDevice(texDevice.GetAddressOf());
if (dev != texDevice.Get())
return false;
// Check if shared texture has correct size and format
CD3D11_TEXTURE2D_DESC inputDesc{};
tex->GetDesc(&inputDesc);
CD3D11_TEXTURE2D_DESC currentDesc{};
m_srcTex->GetDesc(&currentDesc);
if (inputDesc.Format != currentDesc.Format)
return false;
const UINT width = static_cast<UINT>(frameSize.width());
const UINT height = static_cast<UINT>(frameSize.height());
if (currentDesc.Width != width || currentDesc.Height != height)
return false;
return true;
}
bool TextureBridge::recreateSrc(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex, const QSize &frameSize)
{
m_sharedHandle.close();
CD3D11_TEXTURE2D_DESC desc{};
tex->GetDesc(&desc);
const UINT width = static_cast<UINT>(frameSize.width());
const UINT height = static_cast<UINT>(frameSize.height());
CD3D11_TEXTURE2D_DESC texDesc{ desc.Format, width, height };
texDesc.MipLevels = 1;
texDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX | D3D11_RESOURCE_MISC_SHARED_NTHANDLE;
if (dev->CreateTexture2D(&texDesc, nullptr, m_srcTex.ReleaseAndGetAddressOf()) != S_OK)
return false;
ComPtr<IDXGIResource1> res;
if (m_srcTex.As(&res) != S_OK)
return false;
const HRESULT hr =
res->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ, nullptr, &m_sharedHandle);
if (hr != S_OK || !m_sharedHandle)
return false;
if (m_srcTex.As(&m_srcMutex) != S_OK || !m_srcMutex)
return false;
m_destTex = nullptr;
m_destMutex = nullptr;
return true;
}
namespace {
class D3D11TextureHandles : public QVideoFrameTexturesHandles
{
public:
D3D11TextureHandles(TextureConverterBackendPtr &&converterBackend, QRhi *rhi,
ComPtr<ID3D11Texture2D> &&tex)
: m_parentConverterBackend(std::move(converterBackend)), m_owner{ rhi }, m_tex(std::move(tex))
{
}
quint64 textureHandle(QRhi &rhi, int /*plane*/) override
{
if (&rhi != m_owner)
return 0u;
return reinterpret_cast<qint64>(m_tex.Get());
}
private:
TextureConverterBackendPtr m_parentConverterBackend; // ensures the backend is deleted after the texture
QRhi *m_owner = nullptr;
ComPtr<ID3D11Texture2D> m_tex;
};
}
std::shared_ptr<D3D11TextureConverter> D3D11TextureConverter::create(QRhi *rhi)
{
return std::shared_ptr<D3D11TextureConverter>(new D3D11TextureConverter(rhi));
}
D3D11TextureConverter::D3D11TextureConverter(QRhi *rhi)
: TextureConverterBackend(rhi), m_rhiDevice{ GetD3DDevice(rhi) }
{
if (!m_rhiDevice)
return;
m_rhiDevice->GetImmediateContext(m_rhiCtx.GetAddressOf());
}
QVideoFrameTexturesHandlesUPtr
D3D11TextureConverter::createTextureHandles(AVFrame *frame,
QVideoFrameTexturesHandlesUPtr /*oldHandles*/)
{
if (!m_rhiDevice)
return nullptr;
if (!frame || !frame->hw_frames_ctx || frame->format != AV_PIX_FMT_D3D11)
return nullptr;
const auto *ctx = avFrameDeviceContext(frame);
if (!ctx || ctx->type != AV_HWDEVICE_TYPE_D3D11VA)
return nullptr;
const ComPtr<ID3D11Texture2D> ffmpegTex = getAvFrameTexture(frame);
const int index = getAvFramePoolIndex(frame);
if (rhi->backend() == QRhi::D3D11) {
{
const auto *avDeviceCtx = getHwDeviceContext(ctx);
if (!avDeviceCtx)
return nullptr;
// Lock the FFmpeg device context while we copy from FFmpeg's
// frame pool into a shared texture because the underlying ID3D11DeviceContext
// is not thread safe.
avDeviceCtx->lock(avDeviceCtx->lock_ctx);
QScopeGuard autoUnlock([&] { avDeviceCtx->unlock(avDeviceCtx->lock_ctx); });
// Populate the shared texture with one slice from the frame pool, cropping away
// extra surface alignment areas that FFmpeg adds to the textures
QSize frameSize{ frame->width, frame->height };
if (!m_bridge.copyToSharedTex(avDeviceCtx->device, avDeviceCtx->device_context,
ffmpegTex, index, frameSize)) {
return nullptr;
}
}
// Get a copy of the texture on the RHI device
ComPtr<ID3D11Texture2D> output = m_bridge.copyFromSharedTex(m_rhiDevice, m_rhiCtx);
if (!output)
return nullptr;
return std::make_unique<D3D11TextureHandles>(shared_from_this(), rhi, std::move(output));
}
return nullptr;
}
void D3D11TextureConverter::SetupDecoderTextures(AVCodecContext *s)
{
int ret = avcodec_get_hw_frames_parameters(s, s->hw_device_ctx, AV_PIX_FMT_D3D11,
&s->hw_frames_ctx);
if (ret < 0) {
qCDebug(qLcMediaFFmpegHWAccel) << "Failed to allocate HW frames context" << ret;
return;
}
const auto *frames_ctx = reinterpret_cast<const AVHWFramesContext *>(s->hw_frames_ctx->data);
auto *hwctx = static_cast<AVD3D11VAFramesContext *>(frames_ctx->hwctx);
hwctx->MiscFlags = D3D11_RESOURCE_MISC_SHARED;
hwctx->BindFlags = D3D11_BIND_DECODER | D3D11_BIND_SHADER_RESOURCE;
ret = av_hwframe_ctx_init(s->hw_frames_ctx);
if (ret < 0) {
qCDebug(qLcMediaFFmpegHWAccel) << "Failed to initialize HW frames context" << ret;
av_buffer_unref(&s->hw_frames_ctx);
}
}
AVFrameUPtr copyFromHwPoolD3D11(AVFrameUPtr src)
{
if (!src || !src->hw_frames_ctx || src->format != AV_PIX_FMT_D3D11)
return src;
const AVHWDeviceContext *avDevCtx = avFrameDeviceContext(src.get());
if (!avDevCtx || avDevCtx->type != AV_HWDEVICE_TYPE_D3D11VA)
return src;
AVFrameUPtr dest = makeAVFrame();
if (av_frame_copy_props(dest.get(), src.get()) != 0) {
qCCritical(qLcMediaFFmpegHWAccel) << "Unable to copy frame from decoder pool";
return src;
}
const AVD3D11VADeviceContext *hwDevCtx = getHwDeviceContext(avDevCtx);
ComPtr<ID3D11Texture2D> destTex;
{
hwDevCtx->lock(hwDevCtx->lock_ctx);
destTex = copyTexture(hwDevCtx, src.get());
hwDevCtx->unlock(hwDevCtx->lock_ctx);
}
if (!destTex)
return src;
dest->buf[0] = wrapTextureAsBuffer(destTex);
dest->data[0] = reinterpret_cast<uint8_t *>(destTex.Detach());
dest->data[1] = reinterpret_cast<uint8_t *>(0); // This texture is not a texture array
dest->width = src->width;
dest->height = src->height;
dest->format = src->format;
dest->hw_frames_ctx = av_buffer_ref(src->hw_frames_ctx);
return dest;
}
} // namespace QFFmpeg
QT_END_NAMESPACE