mai2/mu3/cm: Integrate UnityDoorstop with segatools (#11)

[UnityDoorstop](https://github.com/NeighTools/UnityDoorstop) is a tool to execute managed code (.NET DLLs) before Unity does, useful for modding frameworks such as BepInEx.

This PR integrates parts of its code into segatools, so loading BepInEx is as simple as adding 2 lines to `segatools.ini`:
```ini
[unity]
targetAssembly=BepInEx\core\BepInEx.Preloader.dll
```

This PR also factors out the Unity path redirection hooks to its own module.

Reviewed-on: Dniel97/segatools#11
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>

unityhook/config.c

@@ -0,0 +1,14 @@
+#include "config.h"
+
+void unity_config_load(struct unity_config *cfg, const wchar_t *filename) {
+    cfg->enable = GetPrivateProfileIntW(L"unity", L"enable", 1, filename);
+
+    GetPrivateProfileStringW(
+        L"unity",
+        L"targetAssembly",
+        L"",
+        cfg->target_assembly,
+        _countof(cfg->target_assembly),
+        filename
+    );
+}

unityhook/config.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <stdbool.h>
+
+#include <windows.h>
+
+struct unity_config {
+    bool enable;
+    wchar_t target_assembly[MAX_PATH];
+};
+
+void unity_config_load(struct unity_config *cfg, const wchar_t *filename);

unityhook/doorstop.c

@@ -0,0 +1,173 @@
+// A simplified version of NeighTools' UnityDoorstop, allowing mod loaders
+// like BepInEx to be loaded into Unity games.
+//
+// SPDX-License-Identifier: CC0
+// https://github.com/NeighTools/UnityDoorstop
+#include <stdbool.h>
+
+#include <pathcch.h>
+#include <psapi.h>
+
+#include "hooklib/procaddr.h"
+#include "util/dprintf.h"
+
+#include "doorstop.h"
+#include "mono.h"
+#include "util.h"
+
+static void * my_mono_jit_init_version(const char *root_domain_name, const char *runtime_version);
+void doorstop_invoke(void *domain);
+
+static char module_name[MAX_PATH];
+static bool doorstop_hook_initted;
+static struct unity_config unity_config;
+static struct hook_symbol unity_mono_syms[] = {
+{
+        .name = "mono_jit_init_version",
+        .patch = my_mono_jit_init_version,
+    }
+};
+
+void doorstop_mono_hook_init(const struct unity_config *cfg, HINSTANCE module) {
+    if (doorstop_hook_initted || cfg->target_assembly[0] == 0) {
+        return;
+    }
+
+    GetModuleBaseNameA(GetCurrentProcess(), module, module_name, MAX_PATH);
+
+    memcpy(&unity_config, cfg, sizeof(*cfg));
+    load_mono_functions(module);
+    proc_addr_table_push(module_name, unity_mono_syms, _countof(unity_mono_syms));
+
+    doorstop_hook_initted = true;
+}
+
+static void * my_mono_jit_init_version(const char *root_domain_name, const char *runtime_version) {
+    dprintf("Unity: Starting Mono domain \"%s\"\n", root_domain_name);
+
+    SetEnvironmentVariableW(L"DOORSTOP_DLL_SEARCH_DIRS", widen(mono_assembly_getrootdir()));
+
+    void* domain = mono_jit_init_version(root_domain_name, runtime_version);
+
+    doorstop_invoke(domain);
+    return domain;
+}
+
+void doorstop_invoke(void* domain) {
+    if (GetEnvironmentVariableW(L"DOORSTOP_INITIALIZED", NULL, 0) != 0) {
+        dprintf("Unity: Doorstop is already initialized.\n");
+        return;
+    }
+
+    SetEnvironmentVariableW(L"DOORSTOP_INITIALIZED", L"TRUE");
+
+    mono_thread_set_main(mono_thread_current());
+
+    if (mono_domain_set_config) {
+#define CONFIG_EXT L".config"
+
+        wchar_t exe_path[MAX_PATH];
+        size_t exe_path_len = GetModuleFileNameW(NULL, exe_path, MAX_PATH);
+        wchar_t *folder_name = wcsdup(exe_path);
+
+        PathCchRemoveFileSpec(folder_name, exe_path_len + 1);
+
+        char *exe_path_n = narrow(exe_path);
+        char *folder_name_n = narrow(folder_name);
+
+        dprintf("Unity: Setting config paths: base dir: %s; config path: %s\n", folder_name_n, exe_path_n);
+
+        mono_domain_set_config(domain, folder_name_n, exe_path_n);
+
+        free(folder_name);
+        free(exe_path_n);
+        free(folder_name_n);
+
+#undef CONFIG_EXT
+    }
+
+    SetEnvironmentVariableW(L"DOORSTOP_INVOKE_DLL_PATH", unity_config.target_assembly);
+
+    char *assembly_dir = mono_assembly_getrootdir();
+    dprintf("Unity: Assembly directory: %s\n", assembly_dir);
+
+    SetEnvironmentVariableA("DOORSTOP_MANAGED_FOLDER_DIR", assembly_dir);
+
+    wchar_t app_path[MAX_PATH];
+    GetModuleFileNameW(NULL, app_path, MAX_PATH);
+    SetEnvironmentVariableW(L"DOORSTOP_PROCESS_PATH", app_path);
+
+    char* dll_path = narrow(unity_config.target_assembly);
+
+    dprintf("Unity: Loading assembly: %s\n", dll_path);
+
+    void* assembly = mono_domain_assembly_open(domain, dll_path);
+
+    if (!assembly) {
+        dprintf("Unity: Failed to load assembly\n");
+        free(dll_path);
+        return;
+    }
+
+    void *image = mono_assembly_get_image(assembly);
+
+    if (!image) {
+        dprintf("Unity: Assembly image doesn't exist\n");
+        free(dll_path);
+        return;
+    }
+
+    void *desc = mono_method_desc_new("*:Main", FALSE);
+    void *method = mono_method_desc_search_in_image(desc, image);
+
+    if (!method) {
+        dprintf("Unity: Assembly does not have a valid entrypoint.\n");
+        free(dll_path);
+        return;
+    }
+
+    void *signature = mono_method_signature(method);
+    UINT32 params = mono_signature_get_param_count(signature);
+    void **args = NULL;
+
+    if (params == 1) {
+        // If there is a parameter, it's most likely a string[].
+        void *args_array = mono_array_new(domain, mono_get_string_class(), 0);
+        args = malloc(sizeof(void*) * 1);
+        args[0] = args_array;
+    }
+
+    dprintf("Unity: Invoking method %p\n", method);
+
+    void *exc = NULL;
+    mono_runtime_invoke(method, NULL, args, &exc);
+
+    if (exc) {
+        dprintf("Unity: Error invoking method!\n");
+
+        void *ex_class = mono_get_exception_class();
+        void *to_string_desc = mono_method_desc_new("*:ToString()", FALSE);
+        void* to_string_method = mono_method_desc_search_in_class(to_string_desc, ex_class);
+
+        mono_method_desc_free(to_string_desc);
+
+        if (to_string_method) {
+            void* real_to_string_method = mono_object_get_virtual_method(exc, to_string_method);
+            void* exc2 = NULL;
+            void* str = mono_runtime_invoke(real_to_string_method, exc, NULL, &exc2);
+
+            if (!exc2) {
+                char* exc_str = mono_string_to_utf8(str);
+                dprintf("Unity: Error message: %s\n", exc_str);
+            }
+        }
+    }
+
+    mono_method_desc_free(desc);
+    free(dll_path);
+
+    if (args) {
+        free(args);
+        args = NULL;
+    }
+}

unityhook/doorstop.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "config.h"
+
+void doorstop_mono_hook_init(const struct unity_config *cfg, HINSTANCE module);

unityhook/hook.c

@@ -0,0 +1,114 @@
+#include <assert.h>
+#include <stdbool.h>
+
+#include "hook/table.h"
+#include "hooklib/path.h"
+#include "util/dprintf.h"
+
+#include "doorstop.h"
+#include "hook.h"
+
+static bool unity_hook_initted;
+static struct unity_config unity_config;
+
+static const wchar_t *target_modules[] = {
+    L"mono.dll",
+    L"mono-2.0-bdwgc.dll",
+    L"cri_ware_unity.dll",
+};
+static const size_t target_modules_len = _countof(target_modules);
+
+static void dll_hook_insert_hooks(HMODULE target);
+
+static HMODULE WINAPI my_LoadLibraryW(const wchar_t *name);
+static HMODULE (WINAPI *next_LoadLibraryW)(const wchar_t *name);
+
+static const struct hook_symbol unity_kernel32_syms[] = {
+    {
+        .name = "LoadLibraryW",
+        .patch = my_LoadLibraryW,
+        .link = (void **) &next_LoadLibraryW,
+    },
+};
+
+void unity_hook_init(const struct unity_config *cfg, HINSTANCE self) {
+    assert(cfg != NULL);
+
+    if (!cfg->enable) {
+        return;
+    }
+
+    if (unity_hook_initted) {
+        return;
+    }
+
+    memcpy(&unity_config, cfg, sizeof(*cfg));
+    dll_hook_insert_hooks(NULL);
+
+    unity_hook_initted = true;
+    dprintf("Unity: Hook enabled.\n");
+}
+
+static void dll_hook_insert_hooks(HMODULE target) {
+    hook_table_apply(
+        target,
+        "kernel32.dll",
+        unity_kernel32_syms,
+        _countof(unity_kernel32_syms));
+}
+
+static HMODULE WINAPI my_LoadLibraryW(const wchar_t *name) {
+    const wchar_t *name_end;
+    const wchar_t *target_module;
+    bool already_loaded;
+    HMODULE result;
+    size_t name_len;
+    size_t target_module_len;
+
+    if (name == NULL) {
+        SetLastError(ERROR_INVALID_PARAMETER);
+
+        return NULL;
+    }
+
+    // Check if the module is already loaded
+    already_loaded = GetModuleHandleW(name) != NULL;
+
+    // Must call the next handler so the DLL reference count is incremented
+    result = next_LoadLibraryW(name);
+
+    if (!already_loaded && result != NULL) {
+        name_len = wcslen(name);
+
+        // mono entrypoint for injecting target_assembly
+        if (GetProcAddress(result, "mono_jit_init_version")) {
+            doorstop_mono_hook_init(&unity_config, result);
+        }
+
+        for (size_t i = 0; i < target_modules_len; i++) {
+            target_module = target_modules[i];
+            target_module_len = wcslen(target_module);
+
+            // Check if the newly loaded library is at least the length of
+            // the name of the target module
+            if (name_len < target_module_len) {
+                continue;
+            }
+
+            name_end = &name[name_len - target_module_len];
+
+            // Check if the name of the newly loaded library is one of the
+            // modules the path hooks should be injected into
+            if (_wcsicmp(name_end, target_module) != 0) {
+                continue;
+            }
+
+            dprintf("Unity: Loaded %S\n", target_module);
+
+            dll_hook_insert_hooks(result);
+            path_hook_insert_hooks(result);
+        }
+    }
+
+    return result;
+}

unityhook/hook.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <windows.h>
+
+#include "config.h"
+
+void unity_hook_init(const struct unity_config *cfg, HINSTANCE self);

unityhook/meson.build

@@ -0,0 +1,20 @@
+unityhook_lib = static_library(
+    'unityhook',
+    include_directories: inc,
+    implicit_include_directories: false,
+    c_pch: '../precompiled.h',
+    dependencies: [
+        capnhook.get_variable('hook_dep'),
+        pathcch_lib
+    ],
+    sources: [
+        'mono.h',
+        'config.c',
+        'config.h',
+        'doorstop.c',
+        'doorstop.h',
+        'hook.c',
+        'hook.h',
+        'util.h'
+    ],
+)

unityhook/mono.h

@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: CC0
+// https://github.com/NeighTools/UnityDoorstop
+#pragma once
+
+#include <windows.h>
+
+// Here we define the pointers to some functions within mono.dll
+// Note to C learners: these are not signature definitions, but rather "variable"
+// definitions with the function pointer type.
+
+// Note: we use void* instead of the real intented structs defined in mono API
+// This way we don't need to include or define any of Mono's structs, which saves space
+// This, obviously, comes with a drawback of not being able to easily access the contents of the structs
+
+void * (*mono_thread_current)();
+void (*mono_thread_set_main)(void *);
+
+void *(*mono_jit_init_version)(const char *root_domain_name, const char *runtime_version);
+void *(*mono_domain_assembly_open)(void *domain, const char *name);
+void *(*mono_assembly_get_image)(void *assembly);
+void *(*mono_runtime_invoke)(void *method, void *obj, void **params, void **exc);
+
+void *(*mono_method_desc_new)(const char *name, int include_namespace);
+void* (*mono_method_desc_search_in_image)(void* desc, void* image);
+void *(*mono_method_desc_search_in_class)(void *desc, void *klass);
+void (*mono_method_desc_free)(void *desc);
+void *(*mono_method_signature)(void *method);
+UINT32 (*mono_signature_get_param_count)(void *sig);
+
+void (*mono_domain_set_config)(void *domain, char *base_dir, char *config_file_name);
+void *(*mono_array_new)(void *domain, void *eclass, uintptr_t n);
+void *(*mono_get_string_class)();
+
+char *(*mono_assembly_getrootdir)();
+
+// Additional funcs to bootstrap custom MONO
+void (*mono_set_dirs)(const char* assembly_dir, const char* config_dir);
+void (*mono_config_parse)(const char* filename);
+void (*mono_set_assemblies_path)(const char* path);
+void *(*mono_object_to_string)(void* obj, void** exc);
+char *(*mono_string_to_utf8)(void* s);
+
+void *(*mono_image_open_from_data_with_name)(void *data, DWORD data_len, int need_copy, void *status, int refonly,
+                                             const char *name);
+
+void* (*mono_get_exception_class)();
+void* (*mono_object_get_virtual_method)(void* obj_raw, void* method);
+
+void* (*mono_jit_parse_options)(int argc, const char** argv);
+
+typedef enum {
+    MONO_DEBUG_FORMAT_NONE,
+    MONO_DEBUG_FORMAT_MONO,
+    /* Deprecated, the mdb debugger is not longer supported. */
+    MONO_DEBUG_FORMAT_DEBUGGER
+} MonoDebugFormat;
+
+void* (*mono_debug_init)(MonoDebugFormat format);
+void* (*mono_debug_domain_create)(void* domain);
+
+/**
+* \brief Loads Mono C API function pointers so that the above definitions can be called.
+* \param mono_lib Mono.dll module.
+*/
+void load_mono_functions(HMODULE mono_lib) {
+    // Enjoy the fact that C allows such sloppy casting
+    // In C++ you would have to cast to the precise function pointer type
+#define GET_MONO_PROC(name) name = (void*)GetProcAddress(mono_lib, #name)
+
+    // Find and assign all our functions that we are going to use
+    GET_MONO_PROC(mono_domain_assembly_open);
+    GET_MONO_PROC(mono_assembly_get_image);
+    GET_MONO_PROC(mono_runtime_invoke);
+    GET_MONO_PROC(mono_jit_init_version);
+    GET_MONO_PROC(mono_method_desc_new);
+    GET_MONO_PROC(mono_method_desc_search_in_class);
+    GET_MONO_PROC(mono_method_desc_search_in_image);
+    GET_MONO_PROC(mono_method_desc_free);
+    GET_MONO_PROC(mono_method_signature);
+    GET_MONO_PROC(mono_signature_get_param_count);
+    GET_MONO_PROC(mono_array_new);
+    GET_MONO_PROC(mono_get_string_class);
+    GET_MONO_PROC(mono_assembly_getrootdir);
+    GET_MONO_PROC(mono_thread_current);
+    GET_MONO_PROC(mono_thread_set_main);
+    GET_MONO_PROC(mono_domain_set_config);
+    GET_MONO_PROC(mono_set_dirs);
+    GET_MONO_PROC(mono_config_parse);
+    GET_MONO_PROC(mono_set_assemblies_path);
+    GET_MONO_PROC(mono_object_to_string);
+    GET_MONO_PROC(mono_string_to_utf8);
+    GET_MONO_PROC(mono_image_open_from_data_with_name);
+    GET_MONO_PROC(mono_get_exception_class);
+    GET_MONO_PROC(mono_object_get_virtual_method);
+    GET_MONO_PROC(mono_jit_parse_options);
+    GET_MONO_PROC(mono_debug_init);
+    GET_MONO_PROC(mono_debug_domain_create);
+
+#undef GET_MONO_PROC
+}

unityhook/util.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <windows.h>
+
+wchar_t *widen(const char *str) {
+    const int reqsz = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0);
+    wchar_t *result = malloc(reqsz * sizeof(wchar_t));
+
+    MultiByteToWideChar(CP_UTF8, 0, str, -1, result, reqsz);
+    return result;
+}
+
+char *narrow(const wchar_t *str) {
+    const int reqsz = WideCharToMultiByte(CP_UTF8, 0, str, -1, NULL, 0, NULL, NULL);
+    char *result = malloc(reqsz * sizeof(char));
+
+    WideCharToMultiByte(CP_UTF8, 0, str, -1, result, reqsz, NULL, NULL);
+    return result;
+}
+