hooklib/dll.c: Add dynamic loader hook mechanism

hooklib/dll.c

@@ -0,0 +1,328 @@
+/* This is general enough to break out into capnhook eventually.
+   Don't introduce util/ dependencies here. */
+
+#include <windows.h>
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+
+#include "hook/table.h"
+
+#include "hooklib/dll.h"
+
+struct dll_hook_reg {
+    const wchar_t *name;
+    HMODULE redir_mod;
+    const struct hook_symbol *syms;
+    size_t nsyms;
+};
+
+/* Helper functions */
+
+static void dll_hook_init(void);
+static HMODULE dll_hook_search_dll(const wchar_t *name);
+
+/* Hook functions */
+
+static HMODULE WINAPI hook_GetModuleHandleA(const char *name);
+static HMODULE WINAPI hook_GetModuleHandleW(const wchar_t *name);
+static HMODULE WINAPI hook_LoadLibraryA(const char *name);
+static HMODULE WINAPI hook_LoadLibraryW(const wchar_t *name);
+static void * WINAPI hook_GetProcAddress(HMODULE mod, const char *name);
+
+/* Link pointers */
+
+static HMODULE WINAPI (*next_GetModuleHandleA)(const char *name);
+static HMODULE WINAPI (*next_GetModuleHandleW)(const wchar_t *name);
+static HMODULE WINAPI (*next_LoadLibraryA)(const char *name);
+static HMODULE WINAPI (*next_LoadLibraryW)(const wchar_t *name);
+static void * WINAPI (*next_GetProcAddress)(HMODULE mod, const char *name);
+
+static const struct hook_symbol dll_loader_syms[] = {
+    {
+        .name   = "GetModuleHandleA",
+        .patch  = hook_GetModuleHandleA,
+        .link   = (void **) &next_GetModuleHandleA,
+    }, {
+        .name   = "GetModuleHandleW",
+        .patch  = hook_GetModuleHandleW,
+        .link   = (void **) &next_GetModuleHandleW,
+    }, {
+        .name   = "LoadLibraryA",
+        .patch  = hook_LoadLibraryA,
+        .link   = (void **) &next_LoadLibraryA,
+    }, {
+        .name   = "LoadLibraryW",
+        .patch  = hook_LoadLibraryW,
+        .link   = (void **) &next_LoadLibraryW,
+    }, {
+        .name   = "GetProcAddress",
+        .patch  = hook_GetProcAddress,
+        .link   = (void **) &next_GetProcAddress,
+    }
+};
+
+static bool dll_hook_initted;
+static CRITICAL_SECTION dll_hook_lock;
+static struct dll_hook_reg *dll_hook_list;
+static size_t dll_hook_count;
+
+HRESULT dll_hook_push(
+        HMODULE redir_mod,
+        const wchar_t *name,
+        const struct hook_symbol *syms,
+        size_t nsyms)
+{
+    struct dll_hook_reg *new_item;
+    struct dll_hook_reg *new_mem;
+    HRESULT hr;
+
+    assert(name != NULL);
+    assert(syms != NULL);
+
+    dll_hook_init();
+
+    EnterCriticalSection(&dll_hook_lock);
+
+    new_mem = realloc(
+            dll_hook_list,
+            (dll_hook_count + 1) * sizeof(struct dll_hook_reg));
+
+    if (new_mem == NULL) {
+        hr = E_OUTOFMEMORY;
+
+        goto end;
+    }
+
+    new_item = &new_mem[dll_hook_count];
+    new_item->name = name;
+    new_item->syms = syms;
+    new_item->nsyms = nsyms;
+
+    dll_hook_list = new_mem;
+    dll_hook_count++;
+    hr = S_OK;
+
+end:
+    LeaveCriticalSection(&dll_hook_lock);
+
+    return hr;
+}
+
+static void dll_hook_init(void)
+{
+    HMODULE kernel32;
+
+    /* Init is not thread safe, because API hooking is not thread safe. */
+
+    if (dll_hook_initted) {
+        return;
+    }
+
+    dll_hook_initted = true;
+    InitializeCriticalSection(&dll_hook_lock);
+
+    /* Protect against the (probably impossible) scenario where nothing in the
+       process imports LoadLibraryW but something imports LoadLibraryA.
+
+       We know something imports GetModuleHandleW because we do, right here.
+
+       (we're about to hook these APIs of course, so we have to set this up
+       before the hooks go in) */
+
+    kernel32 = GetModuleHandleW(L"kernel32.dll");
+    next_LoadLibraryW = (void *) GetProcAddress(kernel32, "LoadLibraryW");
+
+    /* Now we can apply the hook table */
+
+    hook_table_apply(
+            NULL,
+            "kernel32.dll",
+            dll_loader_syms,
+            _countof(dll_loader_syms));
+}
+
+static HMODULE dll_hook_search_dll(const wchar_t *name)
+{
+    HMODULE result;
+    size_t i;
+
+    result = NULL;
+
+    EnterCriticalSection(&dll_hook_lock);
+
+    for (i = 0 ; i < dll_hook_count ; i++) {
+        if (wcsicmp(name, dll_hook_list[i].name) == 0) {
+            result = dll_hook_list[i].redir_mod;
+
+            break;
+        }
+    }
+
+    LeaveCriticalSection(&dll_hook_lock);
+
+    return result;
+}
+
+static HMODULE WINAPI hook_GetModuleHandleA(const char *name)
+{
+    HMODULE result;
+    wchar_t *name_w;
+    size_t name_c;
+
+    if (name == NULL) {
+        return next_GetModuleHandleA(NULL);
+    }
+
+    mbstowcs_s(&name_c, NULL, 0, name, 0);
+    name_w = malloc(name_c * sizeof(wchar_t));
+
+    if (name_w == NULL) {
+        SetLastError(ERROR_OUTOFMEMORY);
+
+        return NULL;
+    }
+
+    mbstowcs_s(NULL, name_w, name_c, name, name_c - 1);
+    result = hook_GetModuleHandleW(name_w);
+    free(name_w);
+
+    return result;
+}
+
+static HMODULE WINAPI hook_GetModuleHandleW(const wchar_t *name)
+{
+    HMODULE result;
+
+    if (name == NULL) {
+        return next_GetModuleHandleW(NULL);
+    }
+
+    result = dll_hook_search_dll(name);
+
+    if (result != NULL) {
+        SetLastError(ERROR_SUCCESS);
+    } else {
+        result = next_GetModuleHandleW(name);
+    }
+
+    return result;
+}
+
+static HMODULE WINAPI hook_LoadLibraryA(const char *name)
+{
+    HMODULE result;
+    wchar_t *name_w;
+    size_t name_c;
+
+    if (name == NULL) {
+        SetLastError(ERROR_INVALID_PARAMETER);
+
+        return NULL;
+    }
+
+    mbstowcs_s(&name_c, NULL, 0, name, 0);
+    name_w = malloc(name_c * sizeof(wchar_t));
+
+    if (name_w == NULL) {
+        SetLastError(ERROR_OUTOFMEMORY);
+
+        return NULL;
+    }
+
+    mbstowcs_s(NULL, name_w, name_c, name, name_c - 1);
+    result = hook_LoadLibraryW(name_w);
+    free(name_w);
+
+    return result;
+}
+
+static HMODULE WINAPI hook_LoadLibraryW(const wchar_t *name)
+{
+    HMODULE result;
+
+    if (name == NULL) {
+        SetLastError(ERROR_INVALID_PARAMETER);
+
+        return NULL;
+    }
+
+    result = dll_hook_search_dll(name);
+
+    if (result != NULL) {
+        SetLastError(ERROR_SUCCESS);
+    } else {
+        result = next_LoadLibraryW(name);
+    }
+
+    return result;
+}
+
+/* TODO LoadLibraryExA, LoadLibraryExW */
+
+static void * WINAPI hook_GetProcAddress(HMODULE mod, const char *name)
+{
+    const struct hook_symbol *syms;
+    uintptr_t ordinal;
+    size_t nsyms;
+    size_t i;
+
+    if (name == NULL) {
+        SetLastError(ERROR_INVALID_PARAMETER);
+
+        return NULL;
+    }
+
+    syms = NULL;
+    nsyms = 0;
+
+    EnterCriticalSection(&dll_hook_lock);
+
+    for (i = 0 ; i < dll_hook_count ; i++) {
+        if (dll_hook_list[i].redir_mod == mod) {
+            syms = dll_hook_list[i].syms;
+            nsyms = dll_hook_list[i].nsyms;
+
+            break;
+        }
+    }
+
+    LeaveCriticalSection(&dll_hook_lock);
+
+    if (syms == NULL) {
+        return next_GetProcAddress(mod, name);
+    }
+
+    ordinal = (uintptr_t) name;
+
+    if (ordinal > 0xFFFF) {
+        /* Import by name */
+
+        for (i = 0 ; i < nsyms ; i++) {
+            if (strcmp(name, syms[i].name) == 0) {
+                break;
+            }
+        }
+    } else {
+        /* Import by ordinal (and name != NULL so ordinal != 0) */
+
+        for (i = 0 ; i < nsyms ; i++) {
+            if (ordinal == syms[i].ordinal) {
+                break;
+            }
+        }
+    }
+
+    if (i < nsyms) {
+        SetLastError(ERROR_SUCCESS);
+
+        return syms[i].patch;
+    } else {
+        /* GetProcAddress sets this error on failure, although of course MSDN
+           does not see fit to document the exact error code. */
+        SetLastError(ERROR_PROC_NOT_FOUND);
+
+        return NULL;
+    }
+}

hooklib/dll.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hook/table.h"
+
+struct dll_symbol {
+    void *ptr;
+    const char *name;
+    uint16_t ordinal;
+};
+
+HRESULT dll_hook_push(
+        HMODULE redir_mod,
+        const wchar_t *name,
+        const struct hook_symbol *syms,
+        size_t nsyms);

hooklib/meson.build

@@ -9,6 +9,8 @@ hooklib_lib = static_library(
     sources : [
         'clock.c',
         'clock.h',
+        'dll.c',
+        'dll.h',
         'fdshark.c',
         'fdshark.h',
         'gfx.c',