geemili
/
SDL
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added support for 0-copy decode and display using D3D11 

FIXME: We need a way to do this that doesn't involve reaching into the D3D11 texture internals
main
Sam Lantinga 2023-10-09 21:02:27 -07:00
parent d830cd140b
commit a842446f62
2 changed files with 251 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/CMakeLists.txt
View File

@ -183,14 +183,13 @@ if(HAVE_LIBUDEV_H)
endif()
include("${SDL3_SOURCE_DIR}/cmake/FindFFmpeg.cmake")
if(FFmpeg_FOUND AND FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60")
if(FFmpeg_FOUND AND (FFmpeg_AVCODEC_VERSION STREQUAL "" OR FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60"))
if(APPLE)
add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c testffmpeg_videotoolbox.m ${icon_bmp_header})
target_link_options(testffmpeg PRIVATE "-Wl,-framework,CoreFoundation" "-Wl,-framework,CoreVideo" "-Wl,-framework,Metal")
else()
add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
endif()
target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES})
if(HAVE_OPENGLES_V2)
#message(STATUS "Enabling EGL support in testffmpeg")
target_compile_definitions(testffmpeg PRIVATE HAVE_EGL)
@ -198,6 +197,7 @@ if(FFmpeg_FOUND AND FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60")
target_link_libraries(testffmpeg PRIVATE OpenGL::EGL)
endif()
endif()
target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES})
else()
message(STATUS "Can't find ffmpeg 6.0 or newer, skipping testffmpeg")
endif()

265
test/testffmpeg.c
View File

@ -40,9 +40,86 @@
#include "testffmpeg_videotoolbox.h"
#endif
#ifdef __WIN32__
#define COBJMACROS
#include <d3d11.h>
struct D3D11_TextureData
{
ID3D11Texture2D *mainTexture;
ID3D11ShaderResourceView *mainTextureResourceView;
ID3D11RenderTargetView *mainTextureRenderTargetView;
ID3D11Texture2D *stagingTexture;
int lockedTexturePositionX;
int lockedTexturePositionY;
D3D11_FILTER scaleMode;
/* YV12 texture support */
SDL_bool yuv;
ID3D11Texture2D *mainTextureU;
ID3D11ShaderResourceView *mainTextureResourceViewU;
ID3D11Texture2D *mainTextureV;
ID3D11ShaderResourceView *mainTextureResourceViewV;
/* NV12 texture support */
SDL_bool nv12;
ID3D11Texture2D *mainTextureNV;
ID3D11ShaderResourceView *mainTextureResourceViewNV;
Uint8 *pixels;
int pitch;
SDL_Rect locked_rect;
};
/* Rendering view state */
typedef struct SDL_RenderViewState
{
int pixel_w;
int pixel_h;
SDL_Rect viewport;
SDL_Rect clip_rect;
SDL_bool clipping_enabled;
SDL_FPoint scale;
} SDL_RenderViewState;
struct SDL_Texture
{
const void *magic;
Uint32 format; /**< The pixel format of the texture */
int access; /**< SDL_TextureAccess */
int w; /**< The width of the texture */
int h; /**< The height of the texture */
int modMode; /**< The texture modulation mode */
SDL_BlendMode blendMode; /**< The texture blend mode */
SDL_ScaleMode scaleMode; /**< The texture scale mode */
SDL_Color color; /**< Texture modulation values */
SDL_RenderViewState view; /**< Target texture view state */
SDL_Renderer *renderer;
/* Support for formats not supported directly by the renderer */
SDL_Texture *native;
void /*SDL_SW_YUVTexture*/ *yuv;
void *pixels;
int pitch;
SDL_Rect locked_rect;
SDL_Surface *locked_surface; /**< Locked region exposed as a SDL surface */
Uint32 last_command_generation; /* last command queue generation this texture was in. */
struct D3D11_TextureData /*void*/ *driverdata; /**< Driver specific texture representation */
void *userdata;
SDL_Texture *prev;
SDL_Texture *next;
};
#endif /* __WIN32__ */
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/hwcontext_d3d11va.h>
#include <libavutil/hwcontext_drm.h>
#include <libavutil/pixdesc.h>
@ -51,6 +128,7 @@
#define WINDOW_WIDTH 640
#define WINDOW_HEIGHT 480
static SDL_Texture *sprite;
static SDL_FRect *positions;
static SDL_FRect *velocities;
@ -71,21 +149,25 @@ static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOESFunc;
#ifdef __APPLE__
static SDL_bool has_videotoolbox_output;
#endif
#ifdef __WIN32__
static ID3D11Device *d3d11_device;
static ID3D11DeviceContext *d3d11_context;
#endif
static int done;
static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
static SDL_bool CreateWindowAndRenderer(Uint32 window_flags, const char *driver)
{
SDL_RendererInfo info;
SDL_bool useEGL = (driver && SDL_strcmp(driver, "opengles2") == 0);
SDL_SetHint(SDL_HINT_RENDER_DRIVER, driver);
if (useEGL) {
SDL_SetHint( SDL_HINT_VIDEO_FORCE_EGL, "1" );
SDL_SetHint( SDL_HINT_RENDER_DRIVER, "opengles2" );
SDL_SetHint(SDL_HINT_VIDEO_FORCE_EGL, "1");
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
} else {
SDL_SetHint( SDL_HINT_VIDEO_FORCE_EGL, "0" );
SDL_SetHint( SDL_HINT_RENDER_DRIVER, NULL );
SDL_SetHint(SDL_HINT_VIDEO_FORCE_EGL, "0");
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, 0);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 1);
@ -127,6 +209,13 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
has_videotoolbox_output = SetupVideoToolboxOutput(renderer);
#endif
#ifdef __WIN32__
d3d11_device = SDL_GetRenderD3D11Device(renderer);
if (d3d11_device) {
ID3D11Device_GetImmediateContext(d3d11_device, &d3d11_context);
}
#endif
return SDL_TRUE;
}
@ -238,6 +327,11 @@ static SDL_bool SupportedPixelFormat(enum AVPixelFormat format)
return SDL_TRUE;
}
#endif
#ifdef __WIN32__
if (d3d11_device && format == AV_PIX_FMT_D3D11) {
return SDL_TRUE;
}
#endif
if (GetTextureFormat(format) != SDL_PIXELFORMAT_UNKNOWN) {
return SDL_TRUE;
@ -245,7 +339,7 @@ static SDL_bool SupportedPixelFormat(enum AVPixelFormat format)
return SDL_FALSE;
}
static enum AVPixelFormat GetPixelFormat(AVCodecContext *s, const enum AVPixelFormat *pix_fmts)
static enum AVPixelFormat GetSupportedPixelFormat(AVCodecContext *s, const enum AVPixelFormat *pix_fmts)
{
const enum AVPixelFormat *p;
@ -296,29 +390,52 @@ static AVCodecContext *OpenVideoStream(AVFormatContext *ic, int stream, const AV
i = 0;
while (!context->hw_device_ctx &&
(config = avcodec_get_hw_config(codec, i++)) != NULL) {
#if 0
SDL_Log("Found %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
#endif
if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
!SupportedPixelFormat(config->pix_fmt)) {
continue;
}
type = AV_HWDEVICE_TYPE_NONE;
while (!context->hw_device_ctx &&
while (!context->hw_device_ctx &&
(type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE) {
if (type != config->device_type) {
continue;
}
result = av_hwdevice_ctx_create(&context->hw_device_ctx, type, NULL, NULL, 0);
if (result < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
if (type == AV_HWDEVICE_TYPE_D3D11VA) {
AVD3D11VADeviceContext *device_context;
context->hw_device_ctx = av_hwdevice_ctx_alloc(type);
device_context = (AVD3D11VADeviceContext *)((AVHWDeviceContext *)context->hw_device_ctx->data)->hwctx;
device_context->device = d3d11_device;
ID3D11Device_AddRef(device_context->device);
device_context->device_context = d3d11_context;
ID3D11DeviceContext_AddRef(device_context->device_context);
result = av_hwdevice_ctx_init(context->hw_device_ctx);
if (result < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
} else {
SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
}
} else {
SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
result = av_hwdevice_ctx_create(&context->hw_device_ctx, type, NULL, NULL, 0);
if (result < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
} else {
SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
}
}
}
}
/* Allow supported hardware accelerated pixel formats */
context->get_format = GetPixelFormat;
context->get_format = GetSupportedPixelFormat;
result = avcodec_open2(context, codec, NULL);
if (result < 0) {
@ -478,6 +595,96 @@ static SDL_bool GetTextureForVAAPIFrame(AVFrame *frame, SDL_Texture **texture)
return result;
}
static SDL_bool GetTextureForD3D11Frame(AVFrame *frame, SDL_Texture **texture)
{
#ifdef __WIN32__
ID3D11Texture2D *pTexture = (ID3D11Texture2D *)frame->data[0];
UINT iSliceIndex = (UINT)(uintptr_t)frame->data[1];
D3D11_TEXTURE2D_DESC desc;
SDL_zero(desc);
ID3D11Texture2D_GetDesc(pTexture, &desc);
if (desc.Format != DXGI_FORMAT_NV12) {
SDL_SetError("Unsupported texture format, expected DXGI_FORMAT_NV12, got %d", desc.Format);
return SDL_FALSE;
}
if (!*texture || (UINT)(*texture)->w != desc.Width || (UINT)(*texture)->h != desc.Height) {
if (*texture) {
SDL_DestroyTexture(*texture);
} else {
/* First time set up for NV12 textures */
SetYUVConversionMode(frame);
}
*texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_NV12, SDL_TEXTUREACCESS_STATIC, desc.Width, desc.Height);
if (!*texture) {
return SDL_FALSE;
}
/* Set up the resource views for this texture */
struct D3D11_TextureData *pTextureData = (*texture)->driverdata;
if (pTextureData->mainTexture) {
ID3D11Texture2D_Release(pTextureData->mainTexture);
pTextureData->mainTexture = NULL;
}
if (pTextureData->mainTextureResourceView) {
ID3D11ShaderResourceView_Release(pTextureData->mainTextureResourceView);
pTextureData->mainTextureResourceView = NULL;
}
if (pTextureData->mainTextureNV) {
ID3D11Texture2D_Release(pTextureData->mainTextureNV);
pTextureData->mainTextureNV = NULL;
}
if (pTextureData->mainTextureResourceViewNV) {
ID3D11ShaderResourceView_Release(pTextureData->mainTextureResourceViewNV);
pTextureData->mainTextureResourceViewNV = NULL;
}
desc.ArraySize = 1;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
HRESULT result = ID3D11Device_CreateTexture2D(d3d11_device, &desc, NULL, &pTextureData->mainTexture);
if (FAILED(result)) {
SDL_SetError("Couldn't create main texture: 0x%x", result);
return SDL_FALSE;
}
pTextureData->mainTextureNV = pTextureData->mainTexture;
ID3D11Texture2D_AddRef(pTextureData->mainTextureNV);
D3D11_SHADER_RESOURCE_VIEW_DESC resourceViewDesc;
resourceViewDesc.Format = DXGI_FORMAT_R8_UNORM;
resourceViewDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D;
resourceViewDesc.Texture2D.MostDetailedMip = 0;
resourceViewDesc.Texture2D.MipLevels = 1;
result = ID3D11Device_CreateShaderResourceView(d3d11_device,
(ID3D11Resource *)pTextureData->mainTexture,
&resourceViewDesc,
&pTextureData->mainTextureResourceView);
if (FAILED(result)) {
SDL_SetError("Couldn't create main texture view: 0x%x", result);
return SDL_FALSE;
}
resourceViewDesc.Format = DXGI_FORMAT_R8G8_UNORM;
result = ID3D11Device_CreateShaderResourceView(d3d11_device,
(ID3D11Resource *)pTextureData->mainTexture,
&resourceViewDesc,
&pTextureData->mainTextureResourceViewNV);
if (FAILED(result)) {
SDL_SetError("Couldn't create secondary texture view: 0x%x", result);
return SDL_FALSE;
}
}
ID3D11DeviceContext_CopySubresourceRegion(d3d11_context, (ID3D11Resource *)(*texture)->driverdata->mainTexture, 0, 0, 0, 0, (ID3D11Resource *)pTexture, iSliceIndex, NULL);
return SDL_TRUE;
#else
return SDL_FALSE;
#endif
}
static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
{
switch (frame->format) {
@ -485,6 +692,8 @@ static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
return GetTextureForVAAPIFrame(frame, texture);
case AV_PIX_FMT_DRM_PRIME:
return GetTextureForDRMFrame(frame, texture);
case AV_PIX_FMT_D3D11:
return GetTextureForD3D11Frame(frame, texture);
default:
return GetTextureForMemoryFrame(frame, texture);
}
@ -708,11 +917,25 @@ int main(int argc, char *argv[])
#endif
#ifdef HAVE_EGL
/* Try to create an EGL compatible window for DRM hardware frame support */
CreateWindow(window_flags, SDL_TRUE);
if (!window) {
CreateWindowAndRenderer(window_flags, "opengles2");
}
#endif
if (!window && !CreateWindow(window_flags, SDL_FALSE)) {
return_code = 2;
goto quit;
#ifdef __APPLE__
if (!window) {
CreateWindowAndRenderer(window_flags, "metal");
}
#endif
#ifdef __WIN32__
if (!window) {
CreateWindowAndRenderer(window_flags, "direct3d11");
}
#endif
if (!window) {
if (!CreateWindowAndRenderer(window_flags, NULL)) {
return_code = 2;
goto quit;
}
}
if (SDL_SetWindowTitle(window, file) < 0) {
@ -876,6 +1099,16 @@ int main(int argc, char *argv[])
quit:
#ifdef __APPLE__
CleanupVideoToolboxOutput();
#endif
#ifdef __WIN32__
if (d3d11_context) {
ID3D11DeviceContext_Release(d3d11_device);
d3d11_context = NULL;
}
if (d3d11_device) {
ID3D11Device_Release(d3d11_device);
d3d11_device = NULL;
}
#endif
SDL_free(positions);
SDL_free(velocities);