geemili
/
SDL
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added support for 0-copy decode and display using Apple VideoToolbox

main
Sam Lantinga 2023-10-09 17:55:06 -07:00
parent 1bf913b29a
commit d830cd140b
4 changed files with 221 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
test/CMakeLists.txt
View File

@ -184,7 +184,12 @@ endif()
include("${SDL3_SOURCE_DIR}/cmake/FindFFmpeg.cmake")
if(FFmpeg_FOUND AND FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60")
add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
if(APPLE)
add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c testffmpeg_videotoolbox.m ${icon_bmp_header})
target_link_options(testffmpeg PRIVATE "-Wl,-framework,CoreFoundation" "-Wl,-framework,CoreVideo" "-Wl,-framework,Metal")
else()
add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header})
endif()
target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES})
if(HAVE_OPENGLES_V2)
#message(STATUS "Enabling EGL support in testffmpeg")

61
test/testffmpeg.c
View File

@ -36,6 +36,10 @@
#endif
#endif
#ifdef __APPLE__
#include "testffmpeg_videotoolbox.h"
#endif
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
@ -64,6 +68,9 @@ static SDL_bool has_EGL_EXT_image_dma_buf_import;
static PFNGLACTIVETEXTUREARBPROC glActiveTextureARBFunc;
static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOESFunc;
#endif
#ifdef __APPLE__
static SDL_bool has_videotoolbox_output;
#endif
static int done;
static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
@ -87,7 +94,7 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 6);
SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 5);
if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, SDL_WINDOW_HIDDEN, &window, &renderer) < 0) {
if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, window_flags, &window, &renderer) < 0) {
return SDL_FALSE;
}
@ -116,6 +123,10 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL)
}
#endif /* HAVE_EGL */
#ifdef __APPLE__
has_videotoolbox_output = SetupVideoToolboxOutput(renderer);
#endif
return SDL_TRUE;
}
@ -222,6 +233,11 @@ static SDL_bool SupportedPixelFormat(enum AVPixelFormat format)
(format == AV_PIX_FMT_VAAPI || format == AV_PIX_FMT_DRM_PRIME)) {
return SDL_TRUE;
}
#ifdef __APPLE__
if (has_videotoolbox_output && format == AV_PIX_FMT_VIDEOTOOLBOX) {
return SDL_TRUE;
}
#endif
if (GetTextureFormat(format) != SDL_PIXELFORMAT_UNKNOWN) {
return SDL_TRUE;
@ -474,11 +490,41 @@ static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
}
}
static void HandleVideoFrame(AVFrame *frame, double pts)
static void DisplayVideoTexture(AVFrame *frame)
{
/* Update the video texture */
GetTextureForFrame(frame, &video_texture);
if (frame->linesize[0] < 0) {
SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL);
} else {
SDL_RenderTexture(renderer, video_texture, NULL, NULL);
}
}
static void DisplayVideoToolbox(AVFrame *frame)
{
#ifdef __APPLE__
SDL_Rect viewport;
SDL_GetRenderViewport(renderer, &viewport);
DisplayVideoToolboxFrame(renderer, frame->data[3], 0, 0, frame->width, frame->height, viewport.x, viewport.y, viewport.w, viewport.h);
#endif
}
static void DisplayVideoFrame(AVFrame *frame)
{
switch (frame->format) {
case AV_PIX_FMT_VIDEOTOOLBOX:
DisplayVideoToolbox(frame);
break;
default:
DisplayVideoTexture(frame);
break;
}
}
static void HandleVideoFrame(AVFrame *frame, double pts)
{
/* Quick and dirty PTS handling */
if (!video_start) {
video_start = SDL_GetTicks();
@ -489,11 +535,7 @@ static void HandleVideoFrame(AVFrame *frame, double pts)
now = (double)(SDL_GetTicks() - video_start) / 1000.0;
}
if (frame->linesize[0] < 0) {
SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL);
} else {
SDL_RenderTexture(renderer, video_texture, NULL, NULL);
}
DisplayVideoFrame(frame);
/* Render any bouncing balls */
MoveSprite();
@ -658,7 +700,7 @@ int main(int argc, char *argv[])
goto quit;
}
window_flags = SDL_WINDOW_HIDDEN;
window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_HIGH_PIXEL_DENSITY;
#ifdef __APPLE__
window_flags |= SDL_WINDOW_METAL;
#elif !defined(__WIN32__)
@ -832,6 +874,9 @@ int main(int argc, char *argv[])
}
return_code = 0;
quit:
#ifdef __APPLE__
CleanupVideoToolboxOutput();
#endif
SDL_free(positions);
SDL_free(velocities);
av_frame_free(&frame);

15
test/testffmpeg_videotoolbox.h Normal file
View File

@ -0,0 +1,15 @@
/*
Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely.
*/
extern SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer);
extern SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH );
extern void CleanupVideoToolboxOutput();

147
test/testffmpeg_videotoolbox.m Normal file
View File

@ -0,0 +1,147 @@
/*
Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely.
*/
#include <SDL3/SDL.h>
#include "testffmpeg_videotoolbox.h"
#include <CoreVideo/CoreVideo.h>
#include <Metal/Metal.h>
#include <QuartzCore/CAMetalLayer.h>
#include <simd/simd.h>
// Metal BT.601 to RGB conversion shader
static NSString *drawMetalShaderSource =
@" using namespace metal;\n"
"\n"
" struct Vertex\n"
" {\n"
" float4 position [[position]];\n"
" float2 texCoords;\n"
" };\n"
"\n"
" constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear);\n"
"\n"
" vertex Vertex draw_vs(constant Vertex *vertices [[ buffer(0) ]], uint vid [[ vertex_id ]])\n"
" {\n"
" return vertices[ vid ];\n"
" }\n"
"\n"
" fragment float4 draw_ps_bt601(Vertex in [[ stage_in ]],\n"
" texture2d<float> textureY [[ texture(0) ]],\n"
" texture2d<float> textureUV [[ texture(1) ]])\n"
" {\n"
" float3 yuv = float3(textureY.sample(s, in.texCoords).r, textureUV.sample(s, in.texCoords).rg);\n"
" float3 rgb;\n"
" yuv += float3(-0.0627451017, -0.501960814, -0.501960814);\n"
" rgb.r = dot(yuv, float3(1.1644, 0.000, 1.596));\n"
" rgb.g = dot(yuv, float3(1.1644, -0.3918, -0.813));\n"
" rgb.b = dot(yuv, float3(1.1644, 2.0172, 0.000));\n"
" return float4(rgb, 1.0);\n"
" }\n"
;
// keep this structure aligned with the proceeding drawMetalShaderSource's struct Vertex
typedef struct Vertex
{
vector_float4 position;
vector_float2 texCoord;
} Vertex;
static void SetVertex(Vertex *vertex, float x, float y, float s, float t)
{
vertex->position[ 0 ] = x;
vertex->position[ 1 ] = y;
vertex->position[ 2 ] = 0.0f;
vertex->position[ 3 ] = 1.0f;
vertex->texCoord[ 0 ] = s;
vertex->texCoord[ 1 ] = t;
}
static CAMetalLayer *metal_layer;
static id<MTLLibrary> library;
static id<MTLRenderPipelineState> video_pipeline;
SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer)
{ @autoreleasepool {
NSError *error;
// Create the metal view
metal_layer = (CAMetalLayer *)SDL_GetRenderMetalLayer(renderer);
if (!metal_layer) {
return SDL_FALSE;
}
// FIXME: Handle other colorspaces besides BT.601
library = [metal_layer.device newLibraryWithSource:drawMetalShaderSource options:nil error:&error];
MTLRenderPipelineDescriptor *videoPipelineDescriptor = [[MTLRenderPipelineDescriptor new] autorelease];
videoPipelineDescriptor.vertexFunction = [library newFunctionWithName:@"draw_vs"];
videoPipelineDescriptor.fragmentFunction = [library newFunctionWithName:@"draw_ps_bt601"];
videoPipelineDescriptor.colorAttachments[ 0 ].pixelFormat = metal_layer.pixelFormat;
video_pipeline = [metal_layer.device newRenderPipelineStateWithDescriptor:videoPipelineDescriptor error:nil];
if (!video_pipeline) {
SDL_SetError("Couldn't create video pipeline");
return SDL_FALSE;
}
return true;
}}
SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH )
{ @autoreleasepool {
CVPixelBufferRef pPixelBuffer = (CVPixelBufferRef)buffer;
size_t nPixelBufferWidth = CVPixelBufferGetWidthOfPlane(pPixelBuffer, 0);
size_t nPixelBufferHeight = CVPixelBufferGetHeightOfPlane(pPixelBuffer, 0);
id<MTLTexture> videoFrameTextureY = nil;
id<MTLTexture> videoFrameTextureUV = nil;
IOSurfaceRef pSurface = CVPixelBufferGetIOSurface(pPixelBuffer);
MTLTextureDescriptor *textureDescriptorY = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm width:nPixelBufferWidth height:nPixelBufferHeight mipmapped:NO];
MTLTextureDescriptor *textureDescriptorUV = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRG8Unorm width:CVPixelBufferGetWidthOfPlane(pPixelBuffer, 1) height:CVPixelBufferGetHeightOfPlane(pPixelBuffer, 1) mipmapped:NO];
videoFrameTextureY = [[metal_layer.device newTextureWithDescriptor:textureDescriptorY iosurface:pSurface plane:0] autorelease];
videoFrameTextureUV = [[metal_layer.device newTextureWithDescriptor:textureDescriptorUV iosurface:pSurface plane:1] autorelease];
float flMinSrcX = ( srcX + 0.5f ) / nPixelBufferWidth;
float flMaxSrcX = ( srcX + srcW + 0.5f ) / nPixelBufferWidth;
float flMinSrcY = ( srcY + 0.5f ) / nPixelBufferHeight;
float flMaxSrcY = ( srcY + srcH + 0.5f ) / nPixelBufferHeight;
int nOutputWidth, nOutputHeight;
nOutputWidth = metal_layer.drawableSize.width;
nOutputHeight = metal_layer.drawableSize.height;
float flMinDstX = 2.0f * ( ( dstX + 0.5f ) / nOutputWidth ) - 1.0f;
float flMaxDstX = 2.0f * ( ( dstX + dstW + 0.5f ) / nOutputWidth ) - 1.0f;
float flMinDstY = 2.0f * ( ( nOutputHeight - dstY - 0.5f ) / nOutputHeight ) - 1.0f;
float flMaxDstY = 2.0f * ( ( nOutputHeight - ( dstY + dstH ) - 0.5f ) / nOutputHeight ) - 1.0f;
Vertex arrVerts[4];
SetVertex(&arrVerts[0], flMinDstX, flMaxDstY, flMinSrcX, flMaxSrcY);
SetVertex(&arrVerts[1], flMinDstX, flMinDstY, flMinSrcX, flMinSrcY);
SetVertex(&arrVerts[2], flMaxDstX, flMaxDstY, flMaxSrcX, flMaxSrcY);
SetVertex(&arrVerts[3], flMaxDstX, flMinDstY, flMaxSrcX, flMinSrcY);
id<MTLRenderCommandEncoder> renderEncoder = (id<MTLRenderCommandEncoder>)SDL_GetRenderMetalCommandEncoder(renderer);
[renderEncoder setRenderPipelineState:video_pipeline];
[renderEncoder setFragmentTexture:videoFrameTextureY atIndex:0];
[renderEncoder setFragmentTexture:videoFrameTextureUV atIndex:1];
[renderEncoder setVertexBytes:arrVerts length:sizeof(arrVerts) atIndex:0];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:SDL_arraysize(arrVerts)];
return SDL_TRUE;
}}
void CleanupVideoToolboxOutput()
{
}