SDL/test/testffmpeg.c

/*
  Copyright (C) 1997-2023 Sam Lantinga <slouken@libsdl.org>

  This software is provided 'as-is', without any express or implied
  warranty.  In no event will the authors be held liable for any damages
  arising from the use of this software.

  Permission is granted to anyone to use this software for any purpose,
  including commercial applications, and to alter it and redistribute it
  freely.
*/
/* Simple program:  Display a video with a sprite bouncing around over it
 *
 * For a more complete video example, see ffplay.c in the ffmpeg sources.
 */

#include <stdlib.h>
#include <time.h>

#include <SDL3/SDL.h>
#include <SDL3/SDL_main.h>

#ifdef HAVE_EGL
#include <SDL3/SDL_opengl.h>
#include <SDL3/SDL_opengles2.h>
#include <SDL3/SDL_egl.h>

#ifndef fourcc_code
#define fourcc_code(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
#endif
#ifndef DRM_FORMAT_R8
#define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ')
#endif
#ifndef DRM_FORMAT_GR88
#define DRM_FORMAT_GR88 fourcc_code('G', 'R', '8', '8')
#endif
#endif

#ifdef __APPLE__
#include "testffmpeg_videotoolbox.h"
#endif

#ifdef __WIN32__
#define COBJMACROS
#include <d3d11.h>

struct D3D11_TextureData
{
    ID3D11Texture2D *mainTexture;
    ID3D11ShaderResourceView *mainTextureResourceView;
    ID3D11RenderTargetView *mainTextureRenderTargetView;
    ID3D11Texture2D *stagingTexture;
    int lockedTexturePositionX;
    int lockedTexturePositionY;
    D3D11_FILTER scaleMode;

    /* YV12 texture support */
    SDL_bool yuv;
    ID3D11Texture2D *mainTextureU;
    ID3D11ShaderResourceView *mainTextureResourceViewU;
    ID3D11Texture2D *mainTextureV;
    ID3D11ShaderResourceView *mainTextureResourceViewV;

    /* NV12 texture support */
    SDL_bool nv12;
    ID3D11Texture2D *mainTextureNV;
    ID3D11ShaderResourceView *mainTextureResourceViewNV;

    Uint8 *pixels;
    int pitch;
    SDL_Rect locked_rect;
};

/* Rendering view state */
typedef struct SDL_RenderViewState
{
    int pixel_w;
    int pixel_h;
    SDL_Rect viewport;
    SDL_Rect clip_rect;
    SDL_bool clipping_enabled;
    SDL_FPoint scale;

} SDL_RenderViewState;

struct SDL_Texture
{
    const void *magic;
    Uint32 format;            /**< The pixel format of the texture */
    int access;               /**< SDL_TextureAccess */
    int w;                    /**< The width of the texture */
    int h;                    /**< The height of the texture */
    int modMode;              /**< The texture modulation mode */
    SDL_BlendMode blendMode;  /**< The texture blend mode */
    SDL_ScaleMode scaleMode;  /**< The texture scale mode */
    SDL_Color color;          /**< Texture modulation values */
    SDL_RenderViewState view; /**< Target texture view state */

    SDL_Renderer *renderer;

    /* Support for formats not supported directly by the renderer */
    SDL_Texture *native;
    void /*SDL_SW_YUVTexture*/ *yuv;
    void *pixels;
    int pitch;
    SDL_Rect locked_rect;
    SDL_Surface *locked_surface; /**< Locked region exposed as a SDL surface */

    Uint32 last_command_generation; /* last command queue generation this texture was in. */

    struct D3D11_TextureData /*void*/ *driverdata; /**< Driver specific texture representation */
    void *userdata;

    SDL_Texture *prev;
    SDL_Texture *next;
};
#endif /* __WIN32__ */

#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/hwcontext_d3d11va.h>
#include <libavutil/hwcontext_drm.h>
#include <libavutil/pixdesc.h>

#include "icon.h"

#define WINDOW_WIDTH  640
#define WINDOW_HEIGHT 480


static SDL_Texture *sprite;
static SDL_FRect *positions;
static SDL_FRect *velocities;
static int sprite_w, sprite_h;
static int num_sprites = 0;

static SDL_Window *window;
static SDL_Renderer *renderer;
static SDL_AudioStream *audio;
static SDL_Texture *video_texture;
static Uint64 video_start;
static SDL_bool has_eglCreateImage;
#ifdef HAVE_EGL
static SDL_bool has_EGL_EXT_image_dma_buf_import;
static PFNGLACTIVETEXTUREARBPROC glActiveTextureARBFunc;
static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOESFunc;
#endif
#ifdef __APPLE__
static SDL_bool has_videotoolbox_output;
#endif
#ifdef __WIN32__
static ID3D11Device *d3d11_device;
static ID3D11DeviceContext *d3d11_context;
#endif
static int done;

static SDL_bool CreateWindowAndRenderer(Uint32 window_flags, const char *driver)
{
    SDL_RendererInfo info;
    SDL_bool useEGL = (driver && SDL_strcmp(driver, "opengles2") == 0);

    SDL_SetHint(SDL_HINT_RENDER_DRIVER, driver);
    if (useEGL) {
        SDL_SetHint(SDL_HINT_VIDEO_FORCE_EGL, "1");
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
    } else {
        SDL_SetHint(SDL_HINT_VIDEO_FORCE_EGL, "0");
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, 0);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
        SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 1);
    }
    SDL_GL_SetAttribute(SDL_GL_RED_SIZE, 5);
    SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 6);
    SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 5);

    if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, window_flags, &window, &renderer) < 0) {
        return SDL_FALSE;
    }

    if (SDL_GetRendererInfo(renderer, &info) == 0) {
        SDL_Log("Created renderer %s\n", info.name);
    }

#ifdef HAVE_EGL
    if (useEGL) {
        const char *extensions = eglQueryString(eglGetCurrentDisplay(), EGL_EXTENSIONS);
        if (SDL_strstr(extensions, "EGL_EXT_image_dma_buf_import") != NULL) {
            has_EGL_EXT_image_dma_buf_import = SDL_TRUE;
        }

        if (SDL_GL_ExtensionSupported("GL_OES_EGL_image")) {
            glEGLImageTargetTexture2DOESFunc = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress("glEGLImageTargetTexture2DOES");
        }

        glActiveTextureARBFunc = (PFNGLACTIVETEXTUREARBPROC)SDL_GL_GetProcAddress("glActiveTextureARB");

        if (has_EGL_EXT_image_dma_buf_import &&
            glEGLImageTargetTexture2DOESFunc &&
            glActiveTextureARBFunc) {
            has_eglCreateImage = SDL_TRUE;
        }
    }
#endif /* HAVE_EGL */

#ifdef __APPLE__
    has_videotoolbox_output = SetupVideoToolboxOutput(renderer);
#endif

#ifdef __WIN32__
    d3d11_device = SDL_GetRenderD3D11Device(renderer);
    if (d3d11_device) {
        ID3D11Device_GetImmediateContext(d3d11_device, &d3d11_context);
    }
#endif

    return SDL_TRUE;
}

static SDL_Texture *CreateTexture(SDL_Renderer *r, unsigned char *data, unsigned int len, int *w, int *h) {
    SDL_Texture *texture = NULL;
    SDL_Surface *surface;
    SDL_RWops *src = SDL_RWFromConstMem(data, len);
    if (src) {
        surface = SDL_LoadBMP_RW(src, SDL_TRUE);
        if (surface) {
            /* Treat white as transparent */
            SDL_SetSurfaceColorKey(surface, SDL_TRUE, SDL_MapRGB(surface->format, 255, 255, 255));

            texture = SDL_CreateTextureFromSurface(r, surface);
            *w = surface->w;
            *h = surface->h;
            SDL_DestroySurface(surface);
        }
    }
    return texture;
}

static void MoveSprite(void)
{
    SDL_Rect viewport;
    SDL_FRect *position, *velocity;
    int i;

    SDL_GetRenderViewport(renderer, &viewport);

    for (i = 0; i < num_sprites; ++i) {
        position = &positions[i];
        velocity = &velocities[i];
        position->x += velocity->x;
        if ((position->x < 0) || (position->x >= (viewport.w - sprite_w))) {
            velocity->x = -velocity->x;
            position->x += velocity->x;
        }
        position->y += velocity->y;
        if ((position->y < 0) || (position->y >= (viewport.h - sprite_h))) {
            velocity->y = -velocity->y;
            position->y += velocity->y;
        }
    }

    /* Blit the sprite onto the screen */
    for (i = 0; i < num_sprites; ++i) {
        position = &positions[i];

        /* Blit the sprite onto the screen */
        SDL_RenderTexture(renderer, sprite, NULL, position);
    }
}

static Uint32 GetTextureFormat(enum AVPixelFormat format)
{
    switch (format) {
    case AV_PIX_FMT_RGB8:
        return SDL_PIXELFORMAT_RGB332;
    case AV_PIX_FMT_RGB444:
        return SDL_PIXELFORMAT_RGB444;
    case AV_PIX_FMT_RGB555:
        return SDL_PIXELFORMAT_RGB555;
    case AV_PIX_FMT_BGR555:
        return SDL_PIXELFORMAT_BGR555;
    case AV_PIX_FMT_RGB565:
        return SDL_PIXELFORMAT_RGB565;
    case AV_PIX_FMT_BGR565:
        return SDL_PIXELFORMAT_BGR565;
    case AV_PIX_FMT_RGB24:
        return SDL_PIXELFORMAT_RGB24;
    case AV_PIX_FMT_BGR24:
        return SDL_PIXELFORMAT_BGR24;
    case AV_PIX_FMT_0RGB32:
        return SDL_PIXELFORMAT_XRGB8888;
    case AV_PIX_FMT_0BGR32:
        return SDL_PIXELFORMAT_XBGR8888;
    case AV_PIX_FMT_NE(RGB0, 0BGR):
        return SDL_PIXELFORMAT_RGBX8888;
    case AV_PIX_FMT_NE(BGR0, 0RGB):
        return SDL_PIXELFORMAT_BGRX8888;
    case AV_PIX_FMT_RGB32:
        return SDL_PIXELFORMAT_ARGB8888;
    case AV_PIX_FMT_RGB32_1:
        return SDL_PIXELFORMAT_RGBA8888;
    case AV_PIX_FMT_BGR32:
        return SDL_PIXELFORMAT_ABGR8888;
    case AV_PIX_FMT_BGR32_1:
        return SDL_PIXELFORMAT_BGRA8888;
    case AV_PIX_FMT_YUV420P:
        return SDL_PIXELFORMAT_IYUV;
    case AV_PIX_FMT_YUYV422:
        return SDL_PIXELFORMAT_YUY2;
    case AV_PIX_FMT_UYVY422:
        return SDL_PIXELFORMAT_UYVY;
    default:
        return SDL_PIXELFORMAT_UNKNOWN;
    }
}

static SDL_bool SupportedPixelFormat(enum AVPixelFormat format)
{
    if (has_eglCreateImage &&
        (format == AV_PIX_FMT_VAAPI || format == AV_PIX_FMT_DRM_PRIME)) {
        return SDL_TRUE;
    }
#ifdef __APPLE__
    if (has_videotoolbox_output && format == AV_PIX_FMT_VIDEOTOOLBOX) {
        return SDL_TRUE;
    }
#endif
#ifdef __WIN32__
    if (d3d11_device && format == AV_PIX_FMT_D3D11) {
        return SDL_TRUE;
    }
#endif

    if (GetTextureFormat(format) != SDL_PIXELFORMAT_UNKNOWN) {
        return SDL_TRUE;
    }
    return SDL_FALSE;
}

static enum AVPixelFormat GetSupportedPixelFormat(AVCodecContext *s, const enum AVPixelFormat *pix_fmts)
{
    const enum AVPixelFormat *p;

    for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
        if (SupportedPixelFormat(*p)) {
            /* We support this format */
            break;
        }
    }

    if (*p == AV_PIX_FMT_NONE) {
        SDL_Log("Couldn't find a supported pixel format:\n");
        for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) {
            SDL_Log("    %s\n", av_get_pix_fmt_name(*p));
        }
    }

    return *p;
}

static AVCodecContext *OpenVideoStream(AVFormatContext *ic, int stream, const AVCodec *codec)
{
    AVStream *st = ic->streams[stream];
    AVCodecParameters *codecpar = st->codecpar;
    AVCodecContext *context;
    const AVCodecHWConfig *config;
    enum AVHWDeviceType type;
    int i;
    int result;

    SDL_Log("Video stream: %s %dx%d\n", avcodec_get_name(codec->id), codecpar->width, codecpar->height);

    context = avcodec_alloc_context3(NULL);
    if (!context) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "avcodec_alloc_context3 failed");
        return NULL;
    }

    result = avcodec_parameters_to_context(context, ic->streams[stream]->codecpar);
    if (result < 0) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "avcodec_parameters_to_context failed: %s\n", av_err2str(result));
        avcodec_free_context(&context);
        return NULL;
    }
    context->pkt_timebase = ic->streams[stream]->time_base;

    /* Look for supported hardware accelerated configurations */
    i = 0;
    while (!context->hw_device_ctx &&
           (config = avcodec_get_hw_config(codec, i++)) != NULL) {
#if 0
        SDL_Log("Found %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
#endif

        if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
            !SupportedPixelFormat(config->pix_fmt)) {
            continue;
        }

        type = AV_HWDEVICE_TYPE_NONE;
        while (!context->hw_device_ctx &&
               (type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE) {
            if (type != config->device_type) {
                continue;
            }

            if (type == AV_HWDEVICE_TYPE_D3D11VA) {
                AVD3D11VADeviceContext *device_context;

                context->hw_device_ctx = av_hwdevice_ctx_alloc(type);

                device_context = (AVD3D11VADeviceContext *)((AVHWDeviceContext *)context->hw_device_ctx->data)->hwctx;
                device_context->device = d3d11_device;
                ID3D11Device_AddRef(device_context->device);
                device_context->device_context = d3d11_context;
                ID3D11DeviceContext_AddRef(device_context->device_context);

                result = av_hwdevice_ctx_init(context->hw_device_ctx);
                if (result < 0) {
                    SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
                } else {
                    SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
                }
            } else {
                result = av_hwdevice_ctx_create(&context->hw_device_ctx, type, NULL, NULL, 0);
                if (result < 0) {
                    SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create hardware device context: %s", av_err2str(result));
                } else {
                    SDL_Log("Using %s hardware acceleration with pixel format %s\n", av_hwdevice_get_type_name(config->device_type), av_get_pix_fmt_name(config->pix_fmt));
                }
            }
        }
    }

    /* Allow supported hardware accelerated pixel formats */
    context->get_format = GetSupportedPixelFormat;

    result = avcodec_open2(context, codec, NULL);
    if (result < 0) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't open codec %s: %s", avcodec_get_name(context->codec_id), av_err2str(result));
        avcodec_free_context(&context);
        return NULL;
    }

    SDL_SetWindowSize(window, codecpar->width, codecpar->height);

    return context;
}

static void SetYUVConversionMode(AVFrame *frame)
{
    SDL_YUV_CONVERSION_MODE mode = SDL_YUV_CONVERSION_AUTOMATIC;
    if (frame && (frame->format == AV_PIX_FMT_YUV420P || frame->format == AV_PIX_FMT_YUYV422 || frame->format == AV_PIX_FMT_UYVY422)) {
        if (frame->color_range == AVCOL_RANGE_JPEG)
            mode = SDL_YUV_CONVERSION_JPEG;
        else if (frame->colorspace == AVCOL_SPC_BT709)
            mode = SDL_YUV_CONVERSION_BT709;
        else if (frame->colorspace == AVCOL_SPC_BT470BG || frame->colorspace == AVCOL_SPC_SMPTE170M)
            mode = SDL_YUV_CONVERSION_BT601;
    }
    SDL_SetYUVConversionMode(mode); /* FIXME: no support for linear transfer */
}

static SDL_bool GetTextureForMemoryFrame(AVFrame *frame, SDL_Texture **texture)
{
    int texture_width = 0, texture_height = 0;
    Uint32 texture_format = SDL_PIXELFORMAT_UNKNOWN;
    Uint32 frame_format = GetTextureFormat(frame->format);

    if (*texture) {
        SDL_QueryTexture(*texture, &texture_format, NULL, &texture_width, &texture_height);
    }
    if (!*texture || frame_format != texture_format || frame->width != texture_width || frame->height != texture_height) {
        if (*texture) {
            SDL_DestroyTexture(*texture);
        }

        *texture = SDL_CreateTexture(renderer, frame_format, SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height);
        if (!*texture) {
            return SDL_FALSE;
        }
    }

    switch (frame_format) {
    case SDL_PIXELFORMAT_IYUV:
        if (frame->linesize[0] > 0 && frame->linesize[1] > 0 && frame->linesize[2] > 0) {
            SDL_UpdateYUVTexture(*texture, NULL, frame->data[0], frame->linesize[0],
                                                   frame->data[1], frame->linesize[1],
                                                   frame->data[2], frame->linesize[2]);
        } else if (frame->linesize[0] < 0 && frame->linesize[1] < 0 && frame->linesize[2] < 0) {
            SDL_UpdateYUVTexture(*texture, NULL, frame->data[0] + frame->linesize[0] * (frame->height                    - 1), -frame->linesize[0],
                                                   frame->data[1] + frame->linesize[1] * (AV_CEIL_RSHIFT(frame->height, 1) - 1), -frame->linesize[1],
                                                   frame->data[2] + frame->linesize[2] * (AV_CEIL_RSHIFT(frame->height, 1) - 1), -frame->linesize[2]);
        }
        SetYUVConversionMode(frame);
        break;
    default:
        if (frame->linesize[0] < 0) {
            SDL_UpdateTexture(*texture, NULL, frame->data[0] + frame->linesize[0] * (frame->height - 1), -frame->linesize[0]);
        } else {
            SDL_UpdateTexture(*texture, NULL, frame->data[0], frame->linesize[0]);
        }
        break;
    }
    return SDL_TRUE;
}

static SDL_bool GetTextureForDRMFrame(AVFrame *frame, SDL_Texture **texture)
{
#ifdef HAVE_EGL
    const AVDRMFrameDescriptor *desc = (const AVDRMFrameDescriptor *)frame->data[0];
    int i, j, image_index, num_planes;
    EGLDisplay display = eglGetCurrentDisplay();

    /* FIXME: Assuming NV12 data format */
    num_planes = 0;
    for (i = 0; i < desc->nb_layers; ++i) {
        num_planes += desc->layers[i].nb_planes;
    }
    if (num_planes != 2) {
        SDL_SetError("Expected NV12 frames with 2 planes, instead got %d planes", num_planes);
        return SDL_FALSE;
    }

    if (*texture) {
        /* Free the previous texture now that we're about to render a new one */
        SDL_DestroyTexture(*texture);
    } else {
        /* First time set up for NV12 textures */
        SDL_SetHint("SDL_RENDER_OPENGL_NV12_RG_SHADER", "1");

        SetYUVConversionMode(frame);
    }

    *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_NV12, SDL_TEXTUREACCESS_STATIC, frame->width, frame->height);
    if (!*texture) {
        return SDL_FALSE;
    }

    /* Bind the texture for importing */
    SDL_GL_BindTexture(*texture, NULL, NULL);

    /* import the frame into OpenGL */
    image_index = 0;
    for (i = 0; i < desc->nb_layers; ++i) {
        const AVDRMLayerDescriptor *layer = &desc->layers[i];
        for (j = 0; j < layer->nb_planes; ++j) {
            static const uint32_t formats[ 2 ] = { DRM_FORMAT_R8, DRM_FORMAT_GR88 };
            const AVDRMPlaneDescriptor *plane = &layer->planes[j];
            const AVDRMObjectDescriptor *object = &desc->objects[plane->object_index];
            EGLAttrib img_attr[] = {
                EGL_LINUX_DRM_FOURCC_EXT,      formats[i],
                EGL_WIDTH,                     frame->width  / ( image_index + 1 ),  /* half size for chroma */
                EGL_HEIGHT,                    frame->height / ( image_index + 1 ),
                EGL_DMA_BUF_PLANE0_FD_EXT,     object->fd,
                EGL_DMA_BUF_PLANE0_OFFSET_EXT, plane->offset,
                EGL_DMA_BUF_PLANE0_PITCH_EXT,  plane->pitch,
                EGL_NONE
            };
            EGLImage pImage = eglCreateImage(display, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, NULL, img_attr);

            glActiveTextureARBFunc(GL_TEXTURE0_ARB + image_index);
            glEGLImageTargetTexture2DOESFunc(GL_TEXTURE_2D, pImage);
            ++image_index;
        }
    }

    SDL_GL_UnbindTexture(*texture);

    return SDL_TRUE;
#else
    return SDL_FALSE;
#endif
}

static SDL_bool GetTextureForVAAPIFrame(AVFrame *frame, SDL_Texture **texture)
{
    AVFrame *drm_frame;
    SDL_bool result = SDL_FALSE;

    drm_frame = av_frame_alloc();
    if (drm_frame) {
        drm_frame->format = AV_PIX_FMT_DRM_PRIME;
        if (av_hwframe_map(drm_frame, frame, 0) == 0) {
            result = GetTextureForDRMFrame(drm_frame, texture);
        } else {
            SDL_SetError("Couldn't map hardware frame");
        }
        av_frame_free(&drm_frame);
    } else {
        SDL_OutOfMemory();
    }
    return result;
}

static SDL_bool GetTextureForD3D11Frame(AVFrame *frame, SDL_Texture **texture)
{
#ifdef __WIN32__
    ID3D11Texture2D *pTexture = (ID3D11Texture2D *)frame->data[0];
    UINT iSliceIndex = (UINT)(uintptr_t)frame->data[1];

    D3D11_TEXTURE2D_DESC desc;
    SDL_zero(desc);
    ID3D11Texture2D_GetDesc(pTexture, &desc);
    if (desc.Format != DXGI_FORMAT_NV12) {
        SDL_SetError("Unsupported texture format, expected DXGI_FORMAT_NV12, got %d", desc.Format);
        return SDL_FALSE;
    }

    if (!*texture || (UINT)(*texture)->w != desc.Width || (UINT)(*texture)->h != desc.Height) {
        if (*texture) {
            SDL_DestroyTexture(*texture);
        } else {
            /* First time set up for NV12 textures */
            SetYUVConversionMode(frame);
        }

        *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_NV12, SDL_TEXTUREACCESS_STATIC, desc.Width, desc.Height);
        if (!*texture) {
            return SDL_FALSE;
        }

        /* Set up the resource views for this texture */
        struct D3D11_TextureData *pTextureData = (*texture)->driverdata;
        if (pTextureData->mainTexture) {
            ID3D11Texture2D_Release(pTextureData->mainTexture);
            pTextureData->mainTexture = NULL;
        }
        if (pTextureData->mainTextureResourceView) {
            ID3D11ShaderResourceView_Release(pTextureData->mainTextureResourceView);
            pTextureData->mainTextureResourceView = NULL;
        }
        if (pTextureData->mainTextureNV) {
            ID3D11Texture2D_Release(pTextureData->mainTextureNV);
            pTextureData->mainTextureNV = NULL;
        }
        if (pTextureData->mainTextureResourceViewNV) {
            ID3D11ShaderResourceView_Release(pTextureData->mainTextureResourceViewNV);
            pTextureData->mainTextureResourceViewNV = NULL;
        }

        desc.ArraySize = 1;
        desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;

        HRESULT result = ID3D11Device_CreateTexture2D(d3d11_device, &desc, NULL, &pTextureData->mainTexture);
        if (FAILED(result)) {
            SDL_SetError("Couldn't create main texture: 0x%x", result);
            return SDL_FALSE;
        }

        pTextureData->mainTextureNV = pTextureData->mainTexture;
        ID3D11Texture2D_AddRef(pTextureData->mainTextureNV);

        D3D11_SHADER_RESOURCE_VIEW_DESC resourceViewDesc;
        resourceViewDesc.Format = DXGI_FORMAT_R8_UNORM;
        resourceViewDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D;
        resourceViewDesc.Texture2D.MostDetailedMip = 0;
        resourceViewDesc.Texture2D.MipLevels = 1;
        result = ID3D11Device_CreateShaderResourceView(d3d11_device,
                                                       (ID3D11Resource *)pTextureData->mainTexture,
                                                       &resourceViewDesc,
                                                       &pTextureData->mainTextureResourceView);
        if (FAILED(result)) {
            SDL_SetError("Couldn't create main texture view: 0x%x", result);
            return SDL_FALSE;
        }

        resourceViewDesc.Format = DXGI_FORMAT_R8G8_UNORM;
        result = ID3D11Device_CreateShaderResourceView(d3d11_device,
                                                       (ID3D11Resource *)pTextureData->mainTexture,
                                                       &resourceViewDesc,
                                                       &pTextureData->mainTextureResourceViewNV);
        if (FAILED(result)) {
            SDL_SetError("Couldn't create secondary texture view: 0x%x", result);
            return SDL_FALSE;
        }
    }

    ID3D11DeviceContext_CopySubresourceRegion(d3d11_context, (ID3D11Resource *)(*texture)->driverdata->mainTexture, 0, 0, 0, 0, (ID3D11Resource *)pTexture, iSliceIndex, NULL);
    return SDL_TRUE;
#else
    return SDL_FALSE;
#endif
}

static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture)
{
    switch (frame->format) {
    case AV_PIX_FMT_VAAPI:
        return GetTextureForVAAPIFrame(frame, texture);
    case AV_PIX_FMT_DRM_PRIME:
        return GetTextureForDRMFrame(frame, texture);
    case AV_PIX_FMT_D3D11:
        return GetTextureForD3D11Frame(frame, texture);
    default:
        return GetTextureForMemoryFrame(frame, texture);
    }
}

static void DisplayVideoTexture(AVFrame *frame)
{
    /* Update the video texture */
    GetTextureForFrame(frame, &video_texture);

    if (frame->linesize[0] < 0) {
        SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL);
    } else {
        SDL_RenderTexture(renderer, video_texture, NULL, NULL);
    }
}

static void DisplayVideoToolbox(AVFrame *frame)
{
#ifdef __APPLE__
    SDL_Rect viewport;
    SDL_GetRenderViewport(renderer, &viewport);
    DisplayVideoToolboxFrame(renderer, frame->data[3], 0, 0, frame->width, frame->height, viewport.x, viewport.y, viewport.w, viewport.h);
#endif
}

static void DisplayVideoFrame(AVFrame *frame)
{
    switch (frame->format) {
    case AV_PIX_FMT_VIDEOTOOLBOX:
        DisplayVideoToolbox(frame);
        break;
    default:
        DisplayVideoTexture(frame);
        break;
    }
}

static void HandleVideoFrame(AVFrame *frame, double pts)
{
    /* Quick and dirty PTS handling */
    if (!video_start) {
        video_start = SDL_GetTicks();
    }
    double now = (double)(SDL_GetTicks() - video_start) / 1000.0;
    while (now < pts - 0.001) {
        SDL_Delay(1);
        now = (double)(SDL_GetTicks() - video_start) / 1000.0;
    }

    DisplayVideoFrame(frame);

    /* Render any bouncing balls */
    MoveSprite();

    SDL_RenderPresent(renderer);
}

static AVCodecContext *OpenAudioStream(AVFormatContext *ic, int stream, const AVCodec *codec)
{
    AVStream *st = ic->streams[stream];
    AVCodecParameters *codecpar = st->codecpar;
    AVCodecContext *context;
    int result;

    SDL_Log("Audio stream: %s %d channels, %d Hz\n", avcodec_get_name(codec->id), codecpar->ch_layout.nb_channels, codecpar->sample_rate);

    context = avcodec_alloc_context3(NULL);
    if (!context) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "avcodec_alloc_context3 failed\n");
        return NULL;
    }

    result = avcodec_parameters_to_context(context, ic->streams[stream]->codecpar);
    if (result < 0) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "avcodec_parameters_to_context failed: %s\n", av_err2str(result));
        avcodec_free_context(&context);
        return NULL;
    }
    context->pkt_timebase = ic->streams[stream]->time_base;

    result = avcodec_open2(context, codec, NULL);
    if (result < 0) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't open codec %s: %s", avcodec_get_name(context->codec_id), av_err2str(result));
        avcodec_free_context(&context);
        return NULL;
    }

    SDL_AudioSpec spec = { SDL_AUDIO_F32, codecpar->ch_layout.nb_channels, codecpar->sample_rate };
    audio = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_OUTPUT, &spec, NULL, NULL);
    if (audio) {
        SDL_ResumeAudioDevice(SDL_GetAudioStreamDevice(audio));
    } else {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't open audio: %s", SDL_GetError());
    }
    return context;
}

static SDL_AudioFormat GetAudioFormat(enum AVSampleFormat format)
{
    switch (format) {
    case AV_SAMPLE_FMT_U8:
    case AV_SAMPLE_FMT_U8P:
        return SDL_AUDIO_U8;
    case AV_SAMPLE_FMT_S16:
    case AV_SAMPLE_FMT_S16P:
        return SDL_AUDIO_S16;
    case AV_SAMPLE_FMT_S32:
    case AV_SAMPLE_FMT_S32P:
        return SDL_AUDIO_S32;
    case AV_SAMPLE_FMT_FLT:
    case AV_SAMPLE_FMT_FLTP:
        return SDL_AUDIO_F32;
    default:
        /* Unsupported */
        return 0;
    }
}

static SDL_bool IsPlanarAudioFormat(enum AVSampleFormat format)
{
    switch (format) {
    case AV_SAMPLE_FMT_U8P:
    case AV_SAMPLE_FMT_S16P:
    case AV_SAMPLE_FMT_S32P:
    case AV_SAMPLE_FMT_FLTP:
    case AV_SAMPLE_FMT_DBLP:
    case AV_SAMPLE_FMT_S64P:
        return SDL_TRUE;
    default:
        return SDL_FALSE;
    }
}

static void InterleaveAudio(AVFrame *frame, const SDL_AudioSpec *spec)
{
    int samplesize = SDL_AUDIO_BYTESIZE(spec->format);
    int framesize = SDL_AUDIO_FRAMESIZE(*spec);
    Uint8 *data = (Uint8 *)SDL_malloc(frame->nb_samples * framesize);
    if (!data) {
        return;
    }

    /* This could be optimized with SIMD and not allocating memory each time */
    for (int c = 0; c < spec->channels; ++c) {
        const Uint8 *src = frame->data[c];
        Uint8 *dst = data + c * samplesize;
        for (int n = frame->nb_samples; n--; ) {
            SDL_memcpy(dst, src, samplesize);
            src += samplesize;
            dst += framesize;
        }
    }
    SDL_PutAudioStreamData(audio, data, frame->nb_samples * framesize);
    SDL_free(data);
}

static void HandleAudioFrame(AVFrame *frame)
{
    if (audio) {
        SDL_AudioSpec spec = { GetAudioFormat(frame->format), frame->ch_layout.nb_channels, frame->sample_rate };
        SDL_SetAudioStreamFormat(audio, &spec, NULL);

        if (frame->ch_layout.nb_channels > 1 && IsPlanarAudioFormat(frame->format)) {
            InterleaveAudio(frame, &spec);
        } else {
            SDL_PutAudioStreamData(audio, frame->data[0], frame->nb_samples * SDL_AUDIO_FRAMESIZE(spec));
        }
    }
}

int main(int argc, char *argv[])
{
    const char *file = NULL;
    AVFormatContext *ic = NULL;
    int audio_stream = -1;
    int video_stream = -1;
    const AVCodec *audio_decoder = NULL;
    const AVCodec *video_decoder = NULL;
    AVCodecContext *audio_context = NULL;
    AVCodecContext *video_context = NULL;
    AVPacket *pkt = NULL;
    AVFrame *frame = NULL;
    double first_pts = -1.0;
    int i;
    int result;
    int return_code = -1;
    Uint32 window_flags;
    SDL_bool flushing = SDL_FALSE;
    SDL_bool decoded = SDL_FALSE;

    /* Enable standard application logging */
    SDL_LogSetPriority(SDL_LOG_CATEGORY_APPLICATION, SDL_LOG_PRIORITY_INFO);

    for (i = 1; i < argc; ++i) {
        if (SDL_strcmp(argv[i], "--sprites") == 0 && argv[i+1]) {
            num_sprites = SDL_atoi(argv[i+1]);
            ++i;
        } else {
            /* We'll try to open this as a media file */
            file = argv[i];
            break;
        }
    }
    if (!file) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Usage: %s [--sprites N] video_file\n", argv[0]);
        return_code = 1;
        goto quit;
    }

    if (SDL_Init(SDL_INIT_AUDIO|SDL_INIT_VIDEO) < 0) {
        return_code = 2;
        goto quit;
    }

    window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_HIGH_PIXEL_DENSITY;
#ifdef __APPLE__
    window_flags |= SDL_WINDOW_METAL;
#elif !defined(__WIN32__)
    window_flags |= SDL_WINDOW_OPENGL;
#endif
#ifdef HAVE_EGL
    /* Try to create an EGL compatible window for DRM hardware frame support */
    if (!window) {
        CreateWindowAndRenderer(window_flags, "opengles2");
    }
#endif
#ifdef __APPLE__
    if (!window) {
        CreateWindowAndRenderer(window_flags, "metal");
    }
#endif
#ifdef __WIN32__
    if (!window) {
        CreateWindowAndRenderer(window_flags, "direct3d11");
    }
#endif
    if (!window) {
        if (!CreateWindowAndRenderer(window_flags, NULL)) {
            return_code = 2;
            goto quit;
        }
    }

    if (SDL_SetWindowTitle(window, file) < 0) {
        SDL_Log("SDL_SetWindowTitle: %s", SDL_GetError());
    }

    /* Open the media file */
    result = avformat_open_input(&ic, file, NULL, NULL);
    if (result < 0) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't open %s: %d", argv[1], result);
        return_code = 4;
        goto quit;
    }
    video_stream = av_find_best_stream(ic, AVMEDIA_TYPE_VIDEO, -1, -1, &video_decoder, 0);
    if (video_stream >= 0) {
        video_context = OpenVideoStream(ic, video_stream, video_decoder);
        if (!video_context) {
            return_code = 4;
            goto quit;
        }
    }
    audio_stream = av_find_best_stream(ic, AVMEDIA_TYPE_AUDIO, -1, video_stream, &audio_decoder, 0);
    if (audio_stream >= 0) {
        audio_context = OpenAudioStream(ic, audio_stream, audio_decoder);
        if (!audio_context) {
            return_code = 4;
            goto quit;
        }
    }
    pkt = av_packet_alloc();
    if (!pkt) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "av_packet_alloc failed");
        return_code = 4;
        goto quit;
    }
    frame = av_frame_alloc();
    if (!frame) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "av_frame_alloc failed");
        return_code = 4;
        goto quit;
    }

    /* Create the sprite */
    sprite = CreateTexture(renderer, icon_bmp, icon_bmp_len, &sprite_w, &sprite_h);

    if (sprite == NULL) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Couldn't create texture (%s)", SDL_GetError());
        return_code = 3;
        goto quit;
    }

    /* Allocate memory for the sprite info */
    positions = (SDL_FRect *)SDL_malloc(num_sprites * sizeof(*positions));
    velocities = (SDL_FRect *)SDL_malloc(num_sprites * sizeof(*velocities));
    if (positions == NULL || velocities == NULL) {
        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Out of memory!\n");
        return_code = 3;
        goto quit;
    }

    /* Position sprites and set their velocities */
    SDL_Rect viewport;
    SDL_GetRenderViewport(renderer, &viewport);
    srand((unsigned int)time(NULL));
    for (i = 0; i < num_sprites; ++i) {
        positions[i].x = (float)(rand() % (viewport.w - sprite_w));
        positions[i].y = (float)(rand() % (viewport.h - sprite_h));
        positions[i].w = (float)sprite_w;
        positions[i].h = (float)sprite_h;
        velocities[i].x = 0.0f;
        velocities[i].y = 0.0f;
        while (!velocities[i].x || !velocities[i].y) {
            velocities[i].x = (float)((rand() % (2 + 1)) - 1);
            velocities[i].y = (float)((rand() % (2 + 1)) - 1);
        }
    }

    /* We're ready to go! */
    SDL_ShowWindow(window);

    /* Main render loop */
    done = 0;

    while (!done) {
        SDL_Event event;

        /* Check for events */
        while (SDL_PollEvent(&event)) {
            if (event.type == SDL_EVENT_QUIT || event.type == SDL_EVENT_KEY_DOWN) {
                done = 1;
            }
        }

        if (!flushing) {
            result = av_read_frame(ic, pkt);
            if (result < 0) {
                SDL_Log("End of stream, finishing decode\n");
                if (audio_context) {
                    avcodec_flush_buffers(audio_context);
                }
                if (video_context) {
                    avcodec_flush_buffers(video_context);
                }
                flushing = SDL_TRUE;
            } else {
                if (pkt->stream_index == audio_stream) {
                    result = avcodec_send_packet(audio_context, pkt);
                    if (result < 0) {
                        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "avcodec_send_packet(audio_context) failed: %s", av_err2str(result));
                    }
                } else if (pkt->stream_index == video_stream) {
                    result = avcodec_send_packet(video_context, pkt);
                    if (result < 0) {
                        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "avcodec_send_packet(video_context) failed: %s", av_err2str(result));
                    }
                }
                av_packet_unref(pkt);
            }
        }

        decoded = SDL_FALSE;
        if (audio_context) {
            while (avcodec_receive_frame(audio_context, frame) >= 0) {
                HandleAudioFrame(frame);
                decoded = SDL_TRUE;
            }
            if (flushing) {
                /* Let SDL know we're done sending audio */
                SDL_FlushAudioStream(audio);
            }
        }
        if (video_context) {
            while (avcodec_receive_frame(video_context, frame) >= 0) {
                double pts = ((double)frame->pts * video_context->pkt_timebase.num) / video_context->pkt_timebase.den;
                if (first_pts < 0.0) {
                    first_pts = pts;
                }
                pts -= first_pts;

                HandleVideoFrame(frame, pts);
                decoded = SDL_TRUE;
            }
        } else {
            /* Update video rendering */
            SDL_SetRenderDrawColor(renderer, 0xA0, 0xA0, 0xA0, 0xFF);
            SDL_RenderClear(renderer);
            MoveSprite();
            SDL_RenderPresent(renderer);
        }

        if (flushing && !decoded) {
            if (SDL_GetAudioStreamQueued(audio) > 0) {
                /* Wait a little bit for the audio to finish */
                SDL_Delay(10);
            } else {
                done = 1;
            }
        }
    }
    return_code = 0;
quit:
#ifdef __APPLE__
    CleanupVideoToolboxOutput();
#endif
#ifdef __WIN32__
    if (d3d11_context) {
        ID3D11DeviceContext_Release(d3d11_device);
        d3d11_context = NULL;
    }
    if (d3d11_device) {
        ID3D11Device_Release(d3d11_device);
        d3d11_device = NULL;
    }
#endif
    SDL_free(positions);
    SDL_free(velocities);
    av_frame_free(&frame);
    av_packet_free(&pkt);
    avcodec_free_context(&audio_context);
    avcodec_free_context(&video_context);
    avformat_close_input(&ic);
    SDL_DestroyRenderer(renderer);
    SDL_DestroyWindow(window);
    SDL_Quit();
    return return_code;
}