segatools/hooklib/gfx/dxgi.c

#include <windows.h>
#include <dxgi.h>

#include <assert.h>
#include <stdlib.h>

#include "hook/com-proxy.h"
#include "hook/table.h"

#include "hooklib/config.h"
#include "hooklib/dll.h"
#include "hooklib/gfx/gfx.h"

#include "util/dprintf.h"

typedef HRESULT (WINAPI *CreateDXGIFactory_t)(REFIID riid, void **factory);
typedef HRESULT (WINAPI *CreateDXGIFactory1_t)(REFIID riid, void **factory);

static HRESULT STDMETHODCALLTYPE my_IDXGIFactory_CreateSwapChain(
        IDXGIFactory *self,
        IUnknown *device,
        DXGI_SWAP_CHAIN_DESC *desc,
        IDXGISwapChain **swapchain);
static HRESULT STDMETHODCALLTYPE my_IDXGIFactory1_CreateSwapChain(
        IDXGIFactory1 *self,
        IUnknown *device,
        DXGI_SWAP_CHAIN_DESC *desc,
        IDXGISwapChain **swapchain);

static struct gfx_config gfx_config;
static CreateDXGIFactory_t next_CreateDXGIFactory;
static CreateDXGIFactory1_t next_CreateDXGIFactory1;

static const struct hook_symbol dxgi_hooks[] = {
    {
        .name = "CreateDXGIFactory",
        .patch = CreateDXGIFactory,
        .link = (void **) &next_CreateDXGIFactory,
    }, {
        .name = "CreateDXGIFactory1",
        .patch = CreateDXGIFactory1,
        .link = (void **) &next_CreateDXGIFactory1,
    },
};

void gfx_dxgi_hook_init(const struct gfx_config *cfg, HINSTANCE self)
{
    HMODULE dxgi;

    assert(cfg != NULL);

    if (!cfg->enable) {
        return;
    }

    memcpy(&gfx_config, cfg, sizeof(*cfg));
    hook_table_apply(NULL, "dxgi.dll", dxgi_hooks, _countof(dxgi_hooks));

    if (next_CreateDXGIFactory == NULL || next_CreateDXGIFactory1 == NULL) {
        dxgi = LoadLibraryW(L"dxgi.dll");

        if (dxgi == NULL) {
            dprintf("DXGI: dxgi.dll not found or failed initialization\n");

            goto fail;
        }

        if (next_CreateDXGIFactory == NULL) {
            next_CreateDXGIFactory = (CreateDXGIFactory_t) GetProcAddress(
                    dxgi,
                    "CreateDXGIFactory");
        }
        if (next_CreateDXGIFactory1 == NULL) {
            next_CreateDXGIFactory1 = (CreateDXGIFactory1_t) GetProcAddress(
                    dxgi,
                    "CreateDXGIFactory1");
        }

        if (next_CreateDXGIFactory == NULL) {
            dprintf("DXGI: CreateDXGIFactory not found in loaded dxgi.dll\n");

            goto fail;
        }
        if (next_CreateDXGIFactory1 == NULL) {
            dprintf("DXGI: CreateDXGIFactory1 not found in loaded dxgi.dll\n");

            goto fail;
        }
    }

    if (self != NULL) {
        dll_hook_push(self, L"dxgi.dll");
    }

    return;

fail:
    if (dxgi != NULL) {
        FreeLibrary(dxgi);
    }
}

HRESULT WINAPI CreateDXGIFactory(REFIID riid, void **factory)
{
    struct com_proxy *proxy;
    IDXGIFactoryVtbl *vtbl;
    IDXGIFactory *api;
    HRESULT hr;

    dprintf("DXGI: CreateDXGIFactory hook hit\n");

    api = NULL;
    hr = next_CreateDXGIFactory(riid, factory);

    if (FAILED(hr)) {
        dprintf("DXGI: CreateDXGIFactory returned %x\n", (int) hr);

        goto fail;
    }

    if (memcmp(riid, &IID_IDXGIFactory, sizeof(*riid)) == 0) {
        api = *factory;
        hr = com_proxy_wrap(&proxy, api, sizeof(*api->lpVtbl));

        if (FAILED(hr)) {
            dprintf("DXGI: com_proxy_wrap returned %x\n", (int) hr);

            goto fail;
        }

        vtbl = proxy->vptr;
        vtbl->CreateSwapChain = my_IDXGIFactory_CreateSwapChain;

        *factory = proxy;
    }

    return hr;

fail:
    if (api != NULL) {
        IDXGIFactory_Release(api);
    }

    return hr;
}

HRESULT WINAPI CreateDXGIFactory1(REFIID riid, void **factory)
{
    struct com_proxy *proxy;
    IDXGIFactory1 *api;
    IDXGIFactory1Vtbl *vtbl;
    HRESULT hr;

    dprintf("DXGI: CreateDXGIFactory1 hook hit\n");

    api = NULL;
    hr = next_CreateDXGIFactory1(riid, factory);

    if (FAILED(hr)) {
        dprintf("DXGI: CreateDXGIFactory1 returned %x\n", (int) hr);

        goto fail;
    }

    if (memcmp(riid, &IID_IDXGIFactory1, sizeof(*riid)) == 0) {
        api = *factory;
        hr = com_proxy_wrap(&proxy, api, sizeof(*api->lpVtbl));

        if (FAILED(hr)) {
            dprintf("DXGI: com_proxy_wrap returned %x\n", (int) hr);

            goto fail;
        }

        vtbl = proxy->vptr;
        vtbl->CreateSwapChain = my_IDXGIFactory1_CreateSwapChain;

        *factory = proxy;
    }

    return hr;

fail:
    if (api != NULL) {
        IDXGIFactory1_Release(api);
    }

    return hr;
}

static HRESULT STDMETHODCALLTYPE my_IDXGIFactory_CreateSwapChain(
        IDXGIFactory *self,
        IUnknown *device,
        DXGI_SWAP_CHAIN_DESC *desc,
        IDXGISwapChain **swapchain)
{
    struct com_proxy *proxy;
    IDXGIFactory *real;
    HWND hwnd;
    LONG width;
    LONG height;

    dprintf("DXGI: IDXGIFactory::CreateSwapChain hook hit\n");

    proxy = com_proxy_downcast(self);
    real = proxy->real;

    if (desc != NULL) {
        desc->Windowed = gfx_config.windowed;

        hwnd = desc->OutputWindow;
        width = desc->BufferDesc.Width;
        height = desc->BufferDesc.Height;
    } else {
        hwnd = NULL;
        width = 0;
        height = 0;
    }

    if (hwnd != NULL) {
        /*
        * Ensure window is maximized to avoid a Windows 10 issue where a
        * fullscreen swap chain is not created because the window is minimized
        * at the time of creation.
        */
        ShowWindow(hwnd, SW_RESTORE);

        if (!gfx_config.framed && width > 0 && height > 0) {
            dprintf("DXGI: Resizing window to %ldx%ld\n", width, height);

            SetWindowLongPtrW(hwnd, GWL_STYLE, WS_POPUP);
            SetWindowLongPtrW(hwnd, GWL_EXSTYLE, WS_EX_TOPMOST);

            SetWindowPos(
                    hwnd,
                    HWND_TOP,
                    0,
                    0,
                    width,
                    height,
                    SWP_FRAMECHANGED | SWP_NOSENDCHANGING);

            ShowWindow(hwnd, SW_SHOWMAXIMIZED);
        }
    }

    return IDXGIFactory_CreateSwapChain(real, device, desc, swapchain);
}

static HRESULT STDMETHODCALLTYPE my_IDXGIFactory1_CreateSwapChain(
        IDXGIFactory1 *self,
        IUnknown *device,
        DXGI_SWAP_CHAIN_DESC *desc,
        IDXGISwapChain **swapchain)
{
    dprintf("DXGI: IDXGIFactory1::CreateSwapChain hook forwarding to my_IDXGIFactory_CreateSwapChain\n");

    return my_IDXGIFactory_CreateSwapChain(
            (IDXGIFactory *) self,
            device,
            desc,
            swapchain);
}