idac: fixed in store battle bug

Reviewed-on: #124

LICENSE.txt

@@ -0,0 +1,13 @@
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                   Version 2, December 2004
+ 
+Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+Everyone is permitted to copy and distribute verbatim or modified
+copies of this license document, and changing it is allowed as long
+as the name is changed.
+ 
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+  TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. You just DO WHAT THE FUCK YOU WANT TO.

changelog.md

@@ -1,9 +1,41 @@
 # Changelog
 Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
 
+## 20240408
+### System
++ Modified the game specific documentation
+
+## 20240407
+### Maimai
++ Support maimai DX International [#118](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/118) (Thanks beerpsi!)
++ Fixed the maimai DX reboot time from config [#120](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/120) (Thanks topty!)
+
+## 20240318
+### CXB
++ Fixing handle_data_shop_list_detail_request for Sunrise S1
+
+## 20240302
+### SAO
++ Fixing new profile creation with right heroes and start VP
++ Fix to the Unanalyzed Log responses returning the wrong rewards
++ Documentation revised
+
+## 20240226
+### CXB
++ Fixing paths for rev.py
++ Changed encoding for handle_data_item_list_icon_request
+
+## 20240202
+### SAO
++ Added reader assets and edited the game specific documentation
+
+## 20240118
+### System
++ Added game version names to the readme
+
 ## 20240109
 ### System
-+ Removed `ADD config config` from dockerfile
++ Removed `ADD config config` from dockerfile [#83](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/83) (Thanks zaphkito!)
 
 ### Aimedb
 + Fixed an error that resulted from trying to scan a banned or locked card

core/__init__.py

@@ -1,6 +1,6 @@
 from core.config import CoreConfig
-from core.allnet import AllnetServlet
-from core.aimedb import AimedbFactory
+from core.allnet import AllnetServlet, BillingServlet
+from core.aimedb import AimedbServlette
 from core.title import TitleServlet
 from core.utils import Utils
 from core.mucha import MuchaServlet

core/adb_handlers/base.py

@@ -102,7 +102,7 @@ class ADBHeader:
         magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id = struct.unpack_from("<5H6sI12s", data)
         head = cls(magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id)
         
-        if head.length != len(data):
+        if head.length > len(data):
             raise ADBHeaderException(f"Length is incorrect! Expect {head.length}, got {len(data)}")
         
         return head

core/aimedb.py

@@ -1,9 +1,7 @@
-from twisted.internet.protocol import Factory, Protocol
 import logging, coloredlogs
 from Crypto.Cipher import AES
-import struct
-from typing import Dict, Tuple, Callable, Union
-from typing_extensions import Final
+from typing import Dict, Tuple, Callable, Union, Optional
+import asyncio
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
@@ -11,15 +9,37 @@ from core.utils import create_sega_auth_key
 from core.data import Data
 from .adb_handlers import *
 
-
-class AimedbProtocol(Protocol):
+class AimedbServlette():
     request_list: Dict[int, Tuple[Callable[[bytes, int], Union[ADBBaseResponse, bytes]], int, str]] = {}
-
-    def __init__(self, core_cfg: CoreConfig) -> None:
-        self.logger = logging.getLogger("aimedb")
-        self.config = core_cfg
+    def __init__(self, core_cfg: CoreConfig) -> None:        
+        self.config = core_cfg        
         self.data = Data(core_cfg)
-        if core_cfg.aimedb.key == "":
+
+        self.logger = logging.getLogger("aimedb")
+        if not hasattr(self.logger, "initted"):
+            log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.config.aimedb.loglevel)
+            coloredlogs.install(
+                level=core_cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initted = True
+
+        if not core_cfg.aimedb.key:
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
 
@@ -40,27 +60,31 @@ class AimedbProtocol(Protocol):
 
         self.register_handler(0x13, 0x14, self.handle_log_ex, 'aime_log_ex')
         self.register_handler(0x64, 0x65, self.handle_hello, 'hello')
-        self.register_handler(0x66, 0, self.handle_goodbye, 'goodbye')
-    
+
     def register_handler(self, cmd: int, resp:int, handler: Callable[[bytes, int], Union[ADBBaseResponse, bytes]], name: str) -> None:
         self.request_list[cmd] = (handler, resp, name)
+    
+    def start(self) -> None:
+        self.logger.info(f"Start on port {self.config.aimedb.port}")
+        addr = self.config.aimedb.listen_address if self.config.aimedb.listen_address else self.config.server.listen_address
+        asyncio.create_task(asyncio.start_server(self.dataReceived, addr, self.config.aimedb.port))
+    
+    async def dataReceived(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
+        self.logger.debug(f"Connection made from {writer.get_extra_info('peername')[0]}")
+        while True:
+            try:
+                data: bytes = await reader.read(4096)
+                if len(data) == 0:
+                    self.logger.debug("Connection closed")
+                    return
+                await self.process_data(data, reader, writer)
+                await writer.drain()
+            except ConnectionResetError as e:
+                self.logger.debug("Connection reset, disconnecting")
+                return
 
-    def append_padding(self, data: bytes):
-        """Appends 0s to the end of the data until it's at the correct size"""
-        length = struct.unpack_from("<H", data, 6)
-        padding_size = length[0] - len(data)
-        data += bytes(padding_size)
-        return data
-
-    def connectionMade(self) -> None:
-        self.logger.debug(f"{self.transport.getPeer().host} Connected")
-
-    def connectionLost(self, reason) -> None:
-        self.logger.debug(
-            f"{self.transport.getPeer().host} Disconnected - {reason.value}"
-        )
-
-    def dataReceived(self, data: bytes) -> None:
+    async def process_data(self, data: bytes, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> Optional[bytes]:
+        addr = writer.get_extra_info('peername')[0]
         cipher = AES.new(self.config.aimedb.key.encode(), AES.MODE_ECB)
 
         try:
@@ -68,9 +92,9 @@ class AimedbProtocol(Protocol):
         
         except Exception as e:
             self.logger.error(f"Failed to decrypt {data.hex()} because {e}")
-            return None
+            return
 
-        self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
+        self.logger.debug(f"{addr} wrote {decrypted.hex()}")
 
         try:
             head = ADBHeader.from_data(decrypted)
@@ -79,7 +103,9 @@ class AimedbProtocol(Protocol):
             self.logger.error(f"Error parsing ADB header: {e}")
             try:    
                 encrypted = cipher.encrypt(ADBBaseResponse().make())
-                self.transport.write(encrypted)
+                writer.write(encrypted)
+                await writer.drain()
+                return
 
             except Exception as e:
                 self.logger.error(f"Failed to encrypt default response because {e}")
@@ -89,46 +115,51 @@ class AimedbProtocol(Protocol):
         if head.keychip_id == "ABCD1234567" or head.store_id == 0xfff0:
             self.logger.warning(f"Request from uninitialized AMLib: {vars(head)}")
 
+        if head.cmd == 0x66:
+            self.logger.info("Goodbye")
+            writer.close()
+            return
+
         handler, resp_code, name = self.request_list.get(head.cmd, (self.handle_default, None, 'default'))
 
         if resp_code is None:
             self.logger.warning(f"No handler for cmd {hex(head.cmd)}")
         
         elif resp_code > 0:
-            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {self.transport.getPeer().host}")
+            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {addr}")
         
-        resp = handler(decrypted, resp_code)
+        resp = await handler(decrypted, resp_code)
 
         if type(resp) == ADBBaseResponse or issubclass(type(resp), ADBBaseResponse):
             resp_bytes = resp.make()
-            if len(resp_bytes) != resp.head.length:
-                resp_bytes = self.append_padding(resp_bytes)
 
         elif type(resp) == bytes:
             resp_bytes = resp
         
         elif resp is None: # Nothing to send, probably a goodbye
+            self.logger.warn(f"None return by handler for {name}")
             return
         
         else:
+            self.logger.error(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
             raise TypeError(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
 
-        try:    
+        try:
             encrypted = cipher.encrypt(resp_bytes)
             self.logger.debug(f"Response {resp_bytes.hex()}")
-            self.transport.write(encrypted)
+            writer.write(encrypted)
 
         except Exception as e:
             self.logger.error(f"Failed to encrypt {resp_bytes.hex()} because {e}")
-        
-    def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
+    
+    async def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
         req = ADBHeader.from_data(data)
         return ADBBaseResponse(resp_code, length, 1, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
 
-    def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        return self.handle_default(data, resp_code)
+    async def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        return await self.handle_default(data, resp_code)
 
-    def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         h = ADBHeader.from_data(data)
         if h.protocol_ver >= 0x3030:
             req = h
@@ -143,12 +174,12 @@ class AimedbProtocol(Protocol):
         # We don't currently support campaigns
         return resp
 
-    def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBLookupRequest(data)
-        user_id = self.data.card.get_user_id_from_card(req.access_code)
-        is_banned = self.data.card.get_card_banned(req.access_code)
-        is_locked = self.data.card.get_card_locked(req.access_code)
-
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
+        
         ret = ADBLookupResponse.from_req(req.head, user_id)
         if is_banned and is_locked:
             ret.head.status = ADBStatus.BAN_SYS_USER
@@ -160,14 +191,17 @@ class AimedbProtocol(Protocol):
         self.logger.info(
             f"access_code {req.access_code} -> user_id {ret.user_id}"
         )
+        
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(req.access_code)
         return ret
 
-    def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBLookupRequest(data)
-        user_id = self.data.card.get_user_id_from_card(req.access_code)
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
 
-        is_banned = self.data.card.get_card_banned(req.access_code)
-        is_locked = self.data.card.get_card_locked(req.access_code)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
 
         ret = ADBLookupExResponse.from_req(req.head, user_id)
         if is_banned and is_locked:
@@ -189,9 +223,11 @@ class AimedbProtocol(Protocol):
                 self.logger.debug(f"Generated auth token {auth_key}")
                 ret.auth_key = auth_key_full
 
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(req.access_code)
         return ret
 
-    def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
         """
         On official, I think a card has to be registered for this to actually work, but 
         I'm making the executive decision to not implement that and just kick back our
@@ -207,7 +243,7 @@ class AimedbProtocol(Protocol):
         )
         return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-    def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
         """
         I've never seen this used.
         """
@@ -215,14 +251,14 @@ class AimedbProtocol(Protocol):
         ac = self.data.card.to_access_code(req.idm)
         
         if self.config.server.allow_user_registration:
-            user_id = self.data.user.create_user()
+            user_id = await self.data.user.create_user()
 
             if user_id is None:
                 self.logger.error("Failed to register user!")
                 user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, ac)
+                card_id = await self.data.card.create_card(user_id, ac)
 
                 if card_id is None:
                     self.logger.error("Failed to register card!")
@@ -237,12 +273,14 @@ class AimedbProtocol(Protocol):
                 f"Registration blocked!: access code {ac} (IDm: {req.idm} PMm: {req.pmm})"
             )
 
+        if user_id > 0:
+            await self.data.card.update_card_last_login(ac)
         return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-    def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
         req = ADBFelicaLookup2Request(data)
         access_code = self.data.card.to_access_code(req.idm)
-        user_id = self.data.card.get_user_id_from_card(access_code=access_code)
+        user_id = await self.data.card.get_user_id_from_card(access_code=access_code)
 
         if user_id is None:
             user_id = -1
@@ -260,10 +298,12 @@ class AimedbProtocol(Protocol):
                 auth_key_full = auth_key.encode() + (b"\0" * auth_key_extra_len)
                 self.logger.debug(f"Generated auth token {auth_key}")
                 resp.auth_key = auth_key_full
-
+        
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(access_code)
         return resp
 
-    def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBCampaignClearRequest(data)
 
         resp = ADBCampaignClearResponse.from_req(req.head)
@@ -271,19 +311,19 @@ class AimedbProtocol(Protocol):
         # We don't support campaign stuff
         return resp
 
-    def handle_register(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_register(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLookupRequest(data)
         user_id = -1
 
         if self.config.server.allow_user_registration:
-            user_id = self.data.user.create_user()
+            user_id = await self.data.user.create_user()
 
             if user_id is None:
                 self.logger.error("Failed to register user!")
                 user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, req.access_code)
+                card_id = await self.data.card.create_card(user_id, req.access_code)
 
                 if card_id is None:
                     self.logger.error("Failed to register card!")
@@ -302,20 +342,23 @@ class AimedbProtocol(Protocol):
         if resp.user_id <= 0:
             resp.head.status = ADBStatus.BAN_SYS # Closest we can get to a "You cannot register"
 
+        else:
+            await self.data.card.update_card_last_login(req.access_code)
+
         return resp
 
     # TODO: Save these in some capacity, as deemed relevant
-    def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
         req = ADBStatusLogRequest(data)
         self.logger.info(f"User {req.aime_id} logged {req.status.name} event")
         return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-    def handle_log(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_log(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLogRequest(data)
         self.logger.info(f"User {req.aime_id} logged {req.status.name} event, credit_ct: {req.credit_ct} bet_ct: {req.bet_ct} won_ct: {req.won_ct}")
         return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-    def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLogExRequest(data)
         strs = []
         self.logger.info(f"Recieved {req.num_logs} or {len(req.logs)} logs")
@@ -324,43 +367,3 @@ class AimedbProtocol(Protocol):
             self.logger.debug(f"User {req.logs[x].aime_id} logged {req.logs[x].status.name} event, credit_ct: {req.logs[x].credit_ct} bet_ct: {req.logs[x].bet_ct} won_ct: {req.logs[x].won_ct}")
         return ADBLogExResponse.from_req(req.head)
 
-    def handle_goodbye(self, data: bytes, resp_code: int) -> None:
-        self.logger.info(f"goodbye from {self.transport.getPeer().host}")
-        self.transport.loseConnection()
-        return
-
-class AimedbFactory(Factory):
-    protocol = AimedbProtocol
-
-    def __init__(self, cfg: CoreConfig) -> None:
-        self.config = cfg
-        log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
-        self.logger = logging.getLogger("aimedb")
-
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
-            when="d",
-            backupCount=10,
-        )
-        fileHandler.setFormatter(log_fmt)
-
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
-
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
-
-        self.logger.setLevel(self.config.aimedb.loglevel)
-        coloredlogs.install(
-            level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
-
-        if self.config.aimedb.key == "":
-            self.logger.error("Please set 'key' field in your config file.")
-            exit(1)
-
-        self.logger.info(f"Ready on port {self.config.aimedb.port}")
-
-    def buildProtocol(self, addr):
-        return AimedbProtocol(self.config)

core/allnet.py

@@ -1,20 +1,24 @@
-from typing import Dict, List, Any, Optional, Tuple, Union, Final
-import logging, coloredlogs
-from logging.handlers import TimedRotatingFileHandler
-from twisted.web.http import Request
-from datetime import datetime
 import pytz
 import base64
 import zlib
 import json
+import yaml
+import logging
+import coloredlogs
+import urllib.parse
+import math
+from typing import Dict, List, Any, Optional, Union, Final
+from logging.handlers import TimedRotatingFileHandler
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
+from starlette.applications import Starlette
+from starlette.routing import Route
+from datetime import datetime
 from enum import Enum
 from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
-from time import strptime
-from os import path
-import urllib.parse
-import math
+from os import path, environ, mkdir, access, W_OK
 
 from .config import CoreConfig
 from .utils import Utils
@@ -90,8 +94,8 @@ class DLI_STATUS(Enum):
             return cls.UNKNOWN
 
 class AllnetServlet:
+    allnet_registry: Dict[str, Any] = {}
     def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
-        super().__init__()
         self.config = core_cfg
         self.config_folder = cfg_folder
         self.data = Data(core_cfg)
@@ -120,25 +124,22 @@ class AllnetServlet:
             )
             self.logger.initialized = True
 
-        plugins = Utils.get_all_titles()
+    def startup(self) -> None:
+        self.logger.info(f"Ready on port {self.config.allnet.port if self.config.allnet.standalone else self.config.server.port}")
+        if not TitleServlet.title_registry:
+            TitleServlet(self.config, self.config_folder)
 
-        if len(plugins) == 0:
-            self.logger.error("No games detected!")
-
-        self.logger.info(
-            f"Serving {len(TitleServlet.title_registry)} game codes port {core_cfg.allnet.port}"
-        )
-
-    def handle_poweron(self, request: Request, _: Dict):
+    async def handle_poweron(self, request: Request):
         request_ip = Utils.get_ip_addr(request)
-        pragma_header = request.getHeader('Pragma')
+        pragma_header = request.headers.get('Pragma', "")
         is_dfi = pragma_header is not None and pragma_header == "DFI"
+        data = await request.body()
         
         try:
             if is_dfi:
-                req_urlencode = self.from_dfi(request.content.getvalue())
+                req_urlencode = self.from_dfi(data)
             else:
-                req_urlencode = request.content.getvalue().decode()
+                req_urlencode = data
 
             req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
@@ -155,7 +156,7 @@ class AllnetServlet:
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
-            return b""
+            return PlainTextResponse()
 
         if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
@@ -166,42 +167,42 @@ class AllnetServlet:
 
         self.logger.debug(f"Allnet request: {vars(req)}")
 
-        machine = self.data.arcade.get_machine(req.serial)        
+        machine = await self.data.arcade.get_machine(req.serial)        
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {req.serial} attempted allnet auth from {request_ip}."
-            self.data.base.log_event(
+            await self.data.base.log_event(
                 "allnet", "ALLNET_AUTH_UNKNOWN_SERIAL", logging.WARN, msg
             )
             self.logger.warning(msg)
 
             resp.stat = ALLNET_STAT.bad_machine.value
             resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
         if machine is not None:
-            arcade = self.data.arcade.get_arcade(machine["arcade"])
+            arcade = await self.data.arcade.get_arcade(machine["arcade"])
             if self.config.server.check_arcade_ip:
                 if arcade["ip"] and arcade["ip"] is not None and arcade["ip"] != req.ip:
                     msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip} (expected {arcade['ip']})."
-                    self.data.base.log_event(
+                    await self.data.base.log_event(
                         "allnet", "ALLNET_AUTH_BAD_IP", logging.ERROR, msg
                     )
                     self.logger.warning(msg)
 
                     resp.stat = ALLNET_STAT.bad_shop.value
                     resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                    return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
                 
                 elif (not arcade["ip"] or arcade["ip"] is None) and self.config.server.strict_ip_checking:
                     msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip}, but arcade {arcade['id']} has no IP set! (strict checking enabled)."
-                    self.data.base.log_event(
+                    await self.data.base.log_event(
                         "allnet", "ALLNET_AUTH_NO_SHOP_IP", logging.ERROR, msg
                     )
                     self.logger.warning(msg)
 
                     resp.stat = ALLNET_STAT.bad_shop.value
                     resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                    return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
 
             country = (
@@ -211,7 +212,7 @@ class AllnetServlet:
                 country = AllnetCountryCode.JAPAN.value
 
             resp.country = country
-            resp.place_id = arcade["id"]
+            resp.place_id = f"{arcade['id']:04X}"
             resp.allnet_id = machine["id"]
             resp.name = arcade["name"] if arcade["name"] is not None else ""
             resp.nickname = arcade["nickname"] if arcade["nickname"] is not None else ""
@@ -238,34 +239,40 @@ class AllnetServlet:
         if req.game_id not in TitleServlet.title_registry:
             if not self.config.server.is_develop:
                 msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
-                self.data.base.log_event(
+                await self.data.base.log_event(
                     "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
                 )
                 self.logger.warning(msg)
 
                 resp.stat = ALLNET_STAT.bad_game.value
                 resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
             else:
                 self.logger.info(
                     f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
                 )
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
-                resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
+                resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/{req.game_id}/{req.ver.replace('.', '')}/"
+                resp.host = f"{self.config.server.hostname}:{self.config.server.port}"
                 
                 resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
                 resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
                 
                 self.logger.debug(f"Allnet response: {resp_str}")
-                return (resp_str + "\n").encode("utf-8")
+                return PlainTextResponse(resp_str + "\n")
 
         
         int_ver = req.ver.replace(".", "")
-        resp.uri, resp.host = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial)
+        try:
+            resp.uri, resp.host = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial)
+        except Exception as e:
+            self.logger.error(f"Error running get_allnet_info for {req.game_id} - {e}")
+            resp.stat = ALLNET_STAT.bad_game.value
+            resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
-        self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
+        await self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
 
         resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
@@ -277,18 +284,19 @@ class AllnetServlet:
             request.responseHeaders.addRawHeader('Pragma', 'DFI')
             return self.to_dfi(resp_str)"""
 
-        return resp_str.encode("utf-8")
+        return PlainTextResponse(resp_str)
 
-    def handle_dlorder(self, request: Request, _: Dict):
+    async def handle_dlorder(self, request: Request):
         request_ip = Utils.get_ip_addr(request)
-        pragma_header = request.getHeader('Pragma')
+        pragma_header = request.headers.get('Pragma', "")
         is_dfi = pragma_header is not None and pragma_header == "DFI"
+        data = await request.body()
         
         try:
             if is_dfi:
-                req_urlencode = self.from_dfi(request.content.getvalue())
+                req_urlencode = self.from_dfi(data)
             else:
-                req_urlencode = request.content.getvalue().decode()
+                req_urlencode = data.decode()
 
             req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
@@ -305,7 +313,7 @@ class AllnetServlet:
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
-            return b""
+            return PlainTextResponse()
 
         self.logger.info(
             f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
@@ -316,54 +324,54 @@ class AllnetServlet:
             not self.config.allnet.allow_online_updates
             or not self.config.allnet.update_cfg_folder
         ):
-            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n")
 
         else:  # TODO: Keychip check
             if path.exists(
                 f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
             ):
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+                resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
 
             if path.exists(
                 f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
             ):
-                resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+                resp.uri += f"|http://{self.config.server.hostname}:{self.config.server.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
 
             self.logger.debug(f"Sending download uri {resp.uri}")
-            self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
+            await self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
 
             res_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
             """if is_dfi:
                 request.responseHeaders.addRawHeader('Pragma', 'DFI')
                 return self.to_dfi(res_str)"""
 
-            return res_str
+            return PlainTextResponse(res_str)
 
-    def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
-        if "file" not in match:
-            return b""
+    async def handle_dlorder_ini(self, request: Request) -> bytes:
+        req_file = request.path_params.get("file", "").replace("%0A", "").replace("\n", "")
 
-        req_file = match["file"].replace("%0A", "")
+        if not req_file:
+            return PlainTextResponse(status_code=404)
 
         if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
             self.logger.info(f"Request for DL INI file {req_file} from {Utils.get_ip_addr(request)} successful")
-            self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
+            await self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
             
-            return open(
-                f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
-            ).read()
+            return PlainTextResponse(open(
+                f"{self.config.allnet.update_cfg_folder}/{req_file}", "r", encoding="utf-8"
+            ).read())
 
         self.logger.info(f"DL INI File {req_file} not found")
-        return b""
+        return PlainTextResponse()
 
-    def handle_dlorder_report(self, request: Request, match: Dict) -> bytes:
-        req_raw = request.content.getvalue()
+    async def handle_dlorder_report(self, request: Request) -> bytes:
+        req_raw = await request.body()
         client_ip = Utils.get_ip_addr(request)
         try:
             req_dict: Dict = json.loads(req_raw)
         except Exception as e:
             self.logger.warning(f"Failed to parse DL Report: {e}")
-            return "NG"
+            return PlainTextResponse("NG")
         
         dl_data_type = DLIMG_TYPE.app
         dl_data = req_dict.get("appimage", {})
@@ -374,24 +382,24 @@ class AllnetServlet:
         
         if dl_data is None or not dl_data:
             self.logger.warning(f"Failed to parse DL Report: Invalid format - contains neither appimage nor optimage")
-            return "NG"
+            return PlainTextResponse("NG")
 
         rep = DLReport(dl_data, dl_data_type)
 
         if not rep.validate():
             self.logger.warning(f"Failed to parse DL Report: Invalid format - {rep.err}")
-            return "NG"
+            return PlainTextResponse("NG")
         
         msg = f"{rep.serial} @ {client_ip} reported {rep.rep_type.name} download state {rep.rf_state.name} for {rep.gd} v{rep.dav}:"\
               f" {rep.tdsc}/{rep.tsc} segments downloaded for working files {rep.wfl} with {rep.dfl if rep.dfl else 'none'} complete."
         
-        self.data.base.log_event("allnet", "DL_REPORT", logging.INFO, msg, dl_data)
+        await self.data.base.log_event("allnet", "DL_REPORT", logging.INFO, msg, dl_data)
         self.logger.info(msg)
 
-        return "OK"
+        return PlainTextResponse("OK")
 
-    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
-        req_data = request.content.getvalue()
+    async def handle_loaderstaterecorder(self, request: Request) -> bytes:
+        req_data = await request.body()
         sections = req_data.decode("utf-8").split("\r\n")
         
         req_dict = dict(urllib.parse.parse_qsl(sections[0]))
@@ -403,130 +411,17 @@ class AllnetServlet:
         ip = Utils.get_ip_addr(request)
 
         if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
-            return "NG".encode()
+            return PlainTextResponse("NG")
 
         self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
-        return "OK".encode()
+        return PlainTextResponse("OK")
     
-    def handle_alive(self, request: Request, match: Dict) -> bytes:
-        return "OK".encode()
+    async def handle_alive(self, request: Request) -> bytes:
+        return PlainTextResponse("OK")
 
-    def handle_billing_request(self, request: Request, _: Dict):
-        req_raw = request.content.getvalue()
-        
-        if request.getHeader('Content-Type') == "application/octet-stream":
-            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
-        else:
-            req_unzip = req_raw
-        
-        req_dict = self.billing_req_to_dict(req_unzip)
-        request_ip = Utils.get_ip_addr(request)
-        
-        if req_dict is None:
-            self.logger.error(f"Failed to parse request {request.content.getvalue()}")
-            return b""
-
-        self.logger.debug(f"request {req_dict}")
-
-        rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
-        signer = PKCS1_v1_5.new(rsa)
-        digest = SHA.new()
-        traces: List[TraceData] = []
-        try:
-            req = BillingInfo(req_dict[0])
-        except KeyError as e:
-            self.logger.error(f"Billing request failed to parse: {e}")
-            return f"result=5&linelimit=&message=field is missing or formatting is incorrect\r\n".encode()
-
-        for x in range(1, len(req_dict)):
-            if not req_dict[x]:
-                continue
-            
-            try:
-                tmp = TraceData(req_dict[x])
-                if tmp.trace_type == TraceDataType.CHARGE:
-                    tmp = TraceDataCharge(req_dict[x])
-                elif tmp.trace_type == TraceDataType.EVENT:
-                    tmp = TraceDataEvent(req_dict[x])
-                elif tmp.trace_type == TraceDataType.CREDIT:
-                    tmp = TraceDataCredit(req_dict[x])
-                
-                traces.append(tmp)
-            
-            except KeyError as e:
-                self.logger.warn(f"Tracelog failed to parse: {e}")
-
-        kc_serial_bytes = req.keychipid.encode()
-        
-
-        machine = self.data.arcade.get_machine(req.keychipid)
-        if machine is None and not self.config.server.allow_unregistered_serials:
-            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
-            self.data.base.log_event(
-                "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
-            )
-            self.logger.warning(msg)
-
-            return f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n".encode()
-
-        msg = (
-            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
-            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
-        )
-        self.logger.info(msg)
-        self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
-        if req.traceleft > 0:
-            self.logger.warn(f"{req.traceleft} unsent tracelogs")
-        kc_playlimit = req.playlimit
-        kc_nearfull = req.nearfull
-
-        while req.playcnt > req.playlimit:
-            kc_playlimit += 1024
-            kc_nearfull += 1024
-
-        playlimit = kc_playlimit
-        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
-
-        digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
-        playlimit_sig = signer.sign(digest).hex()
-
-        digest = SHA.new()
-        digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes)
-        nearfull_sig = signer.sign(digest).hex()
-
-        # TODO: playhistory
-
-        #resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
-        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
-
-        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
-
-        self.logger.debug(f"response {vars(resp)}")
-        if req.traceleft > 0:
-            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
-            return f"result=6&waittime=0&linelimit=20\r\n".encode()
-
-        return resp_str.encode("utf-8")
-
-    def handle_naomitest(self, request: Request, _: Dict) -> bytes:
+    async def handle_naomitest(self, request: Request) -> bytes:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
-        return b"naomi ok"
-
-    def billing_req_to_dict(self, data: bytes):
-        """
-        Parses an billing request string into a python dictionary
-        """
-        try:
-            sections = data.decode("ascii").split("\r\n")
-
-            ret = []
-            for x in sections:
-                ret.append(dict(urllib.parse.parse_qsl(x)))
-            return ret
-
-        except Exception as e:
-            self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
-            return None
+        return PlainTextResponse("naomi ok")
 
     def allnet_req_to_dict(self, data: str) -> Optional[List[Dict[str, Any]]]:
         """
@@ -554,6 +449,150 @@ class AllnetServlet:
         zipped = zlib.compress(unzipped)
         return base64.b64encode(zipped)
 
+class BillingServlet:
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str) -> None:
+        self.config = core_cfg
+        self.config_folder = cfg_folder
+        self.data = Data(core_cfg)
+
+        self.logger = logging.getLogger("billing")
+        if not hasattr(self.logger, "initialized"):
+            log_fmt_str = "[%(asctime)s] Billing | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "billing"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(core_cfg.allnet.loglevel)
+            coloredlogs.install(
+                level=core_cfg.billing.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initialized = True
+    
+    def startup(self) -> None:
+        self.logger.info(f"Ready on port {self.config.billing.port if self.config.billing.standalone else self.config.server.port}")
+
+    def billing_req_to_dict(self, data: bytes):
+        """
+        Parses an billing request string into a python dictionary
+        """
+        try:
+            sections = data.decode("ascii").split("\r\n")
+
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
+
+        except Exception as e:
+            self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
+            return None
+    
+    async def handle_billing_request(self, request: Request):
+        req_raw = await request.body()
+        
+        if request.headers.get('Content-Type', '') == "application/octet-stream":
+            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
+        else:
+            req_unzip = req_raw
+        
+        req_dict = self.billing_req_to_dict(req_unzip)
+        request_ip = Utils.get_ip_addr(request)
+        
+        if req_dict is None:
+            self.logger.error(f"Failed to parse request {req_raw}")
+            return PlainTextResponse()
+
+        self.logger.debug(f"request {req_dict}")
+
+        rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
+        signer = PKCS1_v1_5.new(rsa)
+        digest = SHA.new()
+        traces: List[TraceData] = []
+        try:
+            req = BillingInfo(req_dict[0])
+        except KeyError as e:
+            self.logger.error(f"Billing request failed to parse: {e}")
+            return PlainTextResponse("result=5&linelimit=&message=field is missing or formatting is incorrect\r\n")
+
+        for x in range(1, len(req_dict)):
+            if not req_dict[x]:
+                continue
+            
+            try:
+                tmp = TraceData(req_dict[x])
+                if tmp.trace_type == TraceDataType.CHARGE:
+                    tmp = TraceDataCharge(req_dict[x])
+                elif tmp.trace_type == TraceDataType.EVENT:
+                    tmp = TraceDataEvent(req_dict[x])
+                elif tmp.trace_type == TraceDataType.CREDIT:
+                    tmp = TraceDataCredit(req_dict[x])
+                
+                traces.append(tmp)
+            
+            except KeyError as e:
+                self.logger.warn(f"Tracelog failed to parse: {e}")
+
+        kc_serial_bytes = req.keychipid.encode()
+        
+
+        machine = await self.data.arcade.get_machine(req.keychipid)
+        if machine is None and not self.config.server.allow_unregistered_serials:
+            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
+            await self.data.base.log_event(
+                "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
+            )
+            self.logger.warning(msg)
+
+            return PlainTextResponse(f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n")
+
+        msg = (
+            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
+            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
+        )
+        self.logger.info(msg)
+        await self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
+        if req.traceleft > 0:
+            self.logger.warn(f"{req.traceleft} unsent tracelogs")
+        kc_playlimit = req.playlimit
+        kc_nearfull = req.nearfull
+
+        while req.playcnt > req.playlimit:
+            kc_playlimit += 1024
+            kc_nearfull += 1024
+
+        playlimit = kc_playlimit
+        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
+
+        digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
+        playlimit_sig = signer.sign(digest).hex()
+
+        digest = SHA.new()
+        digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes)
+        nearfull_sig = signer.sign(digest).hex()
+
+        # TODO: playhistory
+
+        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
+
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
+
+        self.logger.debug(f"response {vars(resp)}")
+        if req.traceleft > 0:
+            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
+            return PlainTextResponse("result=6&waittime=0&linelimit=20\r\n")
+        
+        return PlainTextResponse(resp_str)
 
 class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
@@ -613,7 +652,6 @@ class AllnetPowerOnResponse3(AllnetPowerOnResponse):
         self.minute = None
         self.second = None
 
-
 class AllnetPowerOnResponse2(AllnetPowerOnResponse):
     def __init__(self) -> None:
         super().__init__()
@@ -623,7 +661,6 @@ class AllnetPowerOnResponse2(AllnetPowerOnResponse):
         self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
-
 class AllnetDownloadOrderRequest:
     def __init__(self, req: Dict) -> None:
         self.game_id = req.get("game_id", "")
@@ -631,7 +668,6 @@ class AllnetDownloadOrderRequest:
         self.serial = req.get("serial", "")
         self.encode = req.get("encode", "")
 
-
 class AllnetDownloadOrderResponse:
     def __init__(self, stat: int = 1, serial: str = "", uri: str = "") -> None:
         self.stat = stat
@@ -781,7 +817,6 @@ class BillingResponse:
         # playhistory -> YYYYMM/C:...
         # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period
 
-
 class AllnetRequestException(Exception):
     def __init__(self, message="") -> None:
         self.message = message
@@ -849,3 +884,48 @@ class DLReport:
             return False
         
         return True
+
+cfg_dir = environ.get("DIANA_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+billing = BillingServlet(cfg, cfg_dir)
+app_billing = Starlette(
+    cfg.server.is_develop, 
+    [        
+        Route("/request", billing.handle_billing_request, methods=["POST"]),
+        Route("/request/", billing.handle_billing_request, methods=["POST"]),
+    ],
+    on_startup=[billing.startup]
+)
+
+allnet = AllnetServlet(cfg, cfg_dir)
+route_lst = [
+    Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]),
+    Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]),
+    Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]),
+    Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]),
+    Route("/naomitest.html", allnet.handle_naomitest),
+]
+
+if cfg.allnet.allow_online_updates:
+    route_lst += [
+        Route("/report-api/Report", allnet.handle_dlorder_report, methods=["POST"]),
+        Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini),
+    ]
+
+app_allnet = Starlette(
+    cfg.server.is_develop, 
+    route_lst,
+    on_startup=[allnet.startup]
+)

core/app.py

@@ -0,0 +1,92 @@
+import yaml
+import logging
+import coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+from starlette.routing import Route
+from starlette.requests import Request
+from starlette.applications import Starlette
+from starlette.responses import PlainTextResponse
+from os import environ, path, mkdir, W_OK, access
+from typing import List
+
+from core import CoreConfig, TitleServlet, MuchaServlet, AllnetServlet, BillingServlet, AimedbServlette
+from core.frontend import FrontendServlet
+
+async def dummy_rt(request: Request):
+    return PlainTextResponse("Service OK")
+
+cfg_dir = environ.get("ARTEMIS_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+logger = logging.getLogger("core")
+log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
+log_fmt = logging.Formatter(log_fmt_str)
+
+fileHandler = TimedRotatingFileHandler(
+    "{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10
+)
+fileHandler.setFormatter(log_fmt)
+
+consoleHandler = logging.StreamHandler()
+consoleHandler.setFormatter(log_fmt)
+
+logger.addHandler(fileHandler)
+logger.addHandler(consoleHandler)
+
+log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
+logger.setLevel(log_lv)
+coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
+
+logger.info(f"Artemis starting in {'develop' if cfg.server.is_develop else 'production'} mode")
+
+title = TitleServlet(cfg, cfg_dir) # This has to be loaded first to load plugins
+mucha = MuchaServlet(cfg, cfg_dir)
+
+route_lst: List[Route] = [
+    # Mucha
+    Route("/mucha_front/boardauth.do", mucha.handle_boardauth, methods=["POST"]),
+    Route("/mucha_front/updatacheck.do", mucha.handle_updatecheck, methods=["POST"]),
+    Route("/mucha_front/downloadstate.do", mucha.handle_dlstate, methods=["POST"]),
+    # General
+    Route("/", dummy_rt),
+    Route("/robots.txt", FrontendServlet.robots)
+]
+
+if not cfg.billing.standalone:
+    billing = BillingServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/request", billing.handle_billing_request, methods=["POST"]),
+        Route("/request/", billing.handle_billing_request, methods=["POST"]),
+    ]
+
+if not cfg.allnet.standalone:
+    allnet = AllnetServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]),
+        Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]),
+        Route("/naomitest.html", allnet.handle_naomitest),
+    ]
+    
+    if cfg.allnet.allow_online_updates:
+        route_lst += [
+            Route("/report-api/Report", allnet.handle_dlorder_report, methods=["POST"]),
+            Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini),
+        ]
+
+for code, game in title.title_registry.items():
+    route_lst += game.get_routes()
+
+app = Starlette(cfg.server.is_develop, route_lst)

core/config.py

@@ -1,16 +1,48 @@
 import logging, os
 from typing import Any
 
-
 class ServerConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
     def listen_address(self) -> str:
+        """
+        Address Artemis will bind to and listen on
+        """
         return CoreConfig.get_config_field(
             self.__config, "core", "server", "listen_address", default="127.0.0.1"
         )
+    
+    @property
+    def hostname(self) -> str:
+        """
+        Hostname sent to games
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "hostname", default="localhost"
+        )
+    
+    @property
+    def port(self) -> int:
+        """
+        Port the game will listen on
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "port", default=80
+        )
+
+    @property
+    def ssl_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "ssl_key", default="cert/title.key"
+        )
+
+    @property
+    def ssl_cert(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
+        )
 
     @property
     def allow_user_registration(self) -> bool:
@@ -43,9 +75,23 @@ class ServerConfig:
         )
 
     @property
-    def threading(self) -> bool:
+    def proxy_port(self) -> int:
+        """
+        What port the proxy is listening on. This will be sent instead of 'port' if 
+        is_using_proxy is True and this value is non-zero
+        """
         return CoreConfig.get_config_field(
-            self.__config, "core", "server", "threading", default=False
+            self.__config, "core", "server", "proxy_port", default=0
+        )
+
+    @property
+    def proxy_port_ssl(self) -> int:
+        """
+        What port the proxy is listening for secure connections on. This will be sent 
+        instead of 'port' if is_using_proxy is True and this value is non-zero
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "proxy_port_ssl", default=0
         )
 
     @property
@@ -66,7 +112,6 @@ class ServerConfig:
             self.__config, "core", "server", "strict_ip_checking", default=False
         )
 
-
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
@@ -79,36 +124,6 @@ class TitleConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "hostname", default="localhost"
-        )
-
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "port", default=8080
-        )
-        
-    @property
-    def port_ssl(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "port_ssl", default=0
-        )
-
-    @property
-    def ssl_key(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "ssl_key", default="cert/title.key"
-        )
-
-    @property
-    def ssl_cert(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
-        )
-
     @property
     def reboot_start_time(self) -> str:
         return CoreConfig.get_config_field(
@@ -121,7 +136,6 @@ class TitleConfig:
             self.__config, "core", "title", "reboot_end_time", default=""
         )
 
-
 class DatabaseConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
@@ -159,7 +173,7 @@ class DatabaseConfig:
     @property
     def protocol(self) -> str:
         return CoreConfig.get_config_field(
-            self.__config, "core", "database", "type", default="mysql"
+            self.__config, "core", "database", "protocol", default="mysql"
         )
 
     @property
@@ -176,16 +190,6 @@ class DatabaseConfig:
             )
         )
 
-    @property
-    def user_table_autoincrement_start(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config,
-            "core",
-            "database",
-            "user_table_autoincrement_start",
-            default=10000,
-        )
-
     @property
     def enable_memcached(self) -> bool:
         return CoreConfig.get_config_field(
@@ -198,13 +202,12 @@ class DatabaseConfig:
             self.__config, "core", "database", "memcached_host", default="localhost"
         )
 
-
 class FrontendConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
-    def enable(self) -> int:
+    def enable(self) -> bool:
         return CoreConfig.get_config_field(
             self.__config, "core", "frontend", "enable", default=False
         )
@@ -212,7 +215,7 @@ class FrontendConfig:
     @property
     def port(self) -> int:
         return CoreConfig.get_config_field(
-            self.__config, "core", "frontend", "port", default=8090
+            self.__config, "core", "frontend", "port", default=8080
         )
 
     @property
@@ -222,20 +225,23 @@ class FrontendConfig:
                 self.__config, "core", "frontend", "loglevel", default="info"
             )
         )
-
+    
+    @property
+    def secret(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "frontend", "secret", default=""
+        )
 
 class AllnetConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
-    def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(
-            CoreConfig.get_config_field(
-                self.__config, "core", "allnet", "loglevel", default="info"
-            )
+    def standalone(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "standalone", default=False
         )
-
+    
     @property
     def port(self) -> int:
         return CoreConfig.get_config_field(
@@ -243,9 +249,11 @@ class AllnetConfig:
         )
 
     @property
-    def ip_check(self) -> bool:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "allnet", "ip_check", default=False
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "allnet", "loglevel", default="info"
+            )
         )
 
     @property
@@ -260,10 +268,23 @@ class AllnetConfig:
             self.__config, "core", "allnet", "update_cfg_folder", default=""
         )
 
-
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
+    
+    @property
+    def standalone(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "standalone", default=True
+        )
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "billing", "loglevel", default="info"
+            )
+        )
 
     @property
     def port(self) -> int:
@@ -289,11 +310,22 @@ class BillingConfig:
             self.__config, "core", "billing", "signing_key", default="cert/billing.key"
         )
 
-
 class AimedbConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "enable", default=True
+        )
+
+    @property
+    def listen_address(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "listen_address", default=""
+        )
+
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -326,17 +358,10 @@ class AimedbConfig:
             self.__config, "core", "aimedb", "id_lifetime_seconds", default=86400
         )
 
-
 class MuchaConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
-    @property
-    def enable(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "enable", default=False
-        )
-
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -345,13 +370,6 @@ class MuchaConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "hostname", default="localhost"
-        )
-
-
 class CoreConfig(dict):
     def __init__(self) -> None:
         self.server = ServerConfig(self)
@@ -373,6 +391,19 @@ class CoreConfig(dict):
             return logging.DEBUG
         else:
             return logging.INFO
+    
+    @classmethod
+    def loglevel_to_str(cls, level: int) -> str:
+        if level == logging.ERROR:
+            return "error"
+        elif level == logging.WARN:
+            return "warn"
+        elif level == logging.INFO:
+            return "info"
+        elif level == logging.DEBUG:
+            return "debug"
+        else:
+            return "notset"
 
     @classmethod
     def get_config_field(

core/data/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

core/data/alembic/alembic.ini

@@ -0,0 +1,64 @@
+# A generic, single database configuration.
+
+[alembic]
+script_location=.
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to migrations//versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat migrations//versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

core/data/alembic/env.py

@@ -0,0 +1,81 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+from core.data.schema.base import metadata
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    raise Exception('Not implemented or configured!')
+
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    ini_section = config.get_section(config.config_ini_section)
+    overrides = context.get_x_argument(as_dictionary=True)
+    for override in overrides:
+        ini_section[override] = overrides[override]
+
+    connectable = engine_from_config(
+        ini_section,
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+            compare_server_default=True,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

core/data/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

core/data/alembic/versions/6a7e8277763b_gekichu_rating_tables.py

@@ -0,0 +1,56 @@
+"""GekiChu rating tables
+
+Revision ID: 6a7e8277763b
+Revises: d8950c7ce2fc
+Create Date: 2024-03-13 12:18:53.210018
+
+"""
+from alembic import op
+from sqlalchemy import Column, Integer, String
+
+
+# revision identifiers, used by Alembic.
+revision = '6a7e8277763b'
+down_revision = 'd8950c7ce2fc'
+branch_labels = None
+depends_on = None
+
+GEKICHU_RATING_TABLE_NAMES = [
+    "chuni_profile_rating",
+    "ongeki_profile_rating",
+]
+
+def upgrade():
+    for table_name in GEKICHU_RATING_TABLE_NAMES:
+        op.create_table(
+            table_name,
+            Column("id", Integer, primary_key=True, nullable=False),
+            Column("user", Integer, nullable=False),
+            Column("version", Integer, nullable=False),
+            Column("type", String(255), nullable=False),
+            Column("index", Integer, nullable=False),
+            Column("musicId", Integer),
+            Column("difficultId", Integer),
+            Column("romVersionCode", Integer),
+            Column("score", Integer),
+            mysql_charset="utf8mb4",
+        )
+        op.create_foreign_key(
+            None,
+            table_name,
+            "aime_user",
+            ["user"],
+            ["id"],
+            ondelete="cascade",
+            onupdate="cascade",
+        )
+        op.create_unique_constraint(
+            f"{table_name}_uk",
+            table_name,
+            ["user", "version", "type", "index"],
+        )
+
+
+def downgrade():
+    for table_name in GEKICHU_RATING_TABLE_NAMES:
+        op.drop_table(table_name)

core/data/alembic/versions/81e44dd6047a_mai2_buddies_support.py

@@ -0,0 +1,68 @@
+"""mai2_buddies_support
+
+Revision ID: 81e44dd6047a
+Revises: d8950c7ce2fc
+Create Date: 2024-03-12 19:10:37.063907
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "81e44dd6047a"
+down_revision = "6a7e8277763b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "mai2_playlog_2p",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user", sa.Integer(), nullable=False),
+        sa.Column("userId1", sa.Integer(), nullable=True),
+        sa.Column("userId2", sa.Integer(), nullable=True),
+        sa.Column("userName1", sa.String(length=25), nullable=True),
+        sa.Column("userName2", sa.String(length=25), nullable=True),
+        sa.Column("regionId", sa.Integer(), nullable=True),
+        sa.Column("placeId", sa.Integer(), nullable=True),
+        sa.Column("user2pPlaylogDetailList", sa.JSON(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["user"], ["aime_user.id"], onupdate="cascade", ondelete="cascade"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        mysql_charset="utf8mb4",
+    )
+
+    op.add_column(
+        "mai2_playlog",
+        sa.Column(
+            "extBool1", sa.Boolean(), nullable=True, server_default=sa.text("NULL")
+        ),
+    )
+
+    op.add_column(
+        "mai2_profile_detail",
+        sa.Column(
+            "renameCredit", sa.Integer(), nullable=True, server_default=sa.text("NULL")
+        ),
+    )
+    op.add_column(
+        "mai2_profile_detail",
+        sa.Column(
+            "currentPlayCount",
+            sa.Integer(),
+            nullable=True,
+            server_default=sa.text("NULL"),
+        ),
+    )
+
+
+def downgrade():
+    op.drop_table("mai2_playlog_2p")
+
+    op.drop_column("mai2_playlog", "extBool1")
+    op.drop_column("mai2_profile_detail", "renameCredit")
+    op.drop_column("mai2_profile_detail", "currentPlayCount")

core/data/alembic/versions/835b862f9bf0_initial_migration.py

@@ -0,0 +1,24 @@
+"""Initial Migration
+
+Revision ID: 835b862f9bf0
+Revises: 
+Create Date: 2024-01-09 13:06:10.787432
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '835b862f9bf0'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

core/data/alembic/versions/d8950c7ce2fc_remove_old_db_mgmt_system.py

@@ -0,0 +1,29 @@
+"""Remove old db mgmt system
+
+Revision ID: d8950c7ce2fc
+Revises: 835b862f9bf0
+Create Date: 2024-01-09 13:43:51.381175
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd8950c7ce2fc'
+down_revision = '835b862f9bf0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_table("schema_versions")
+
+
+def downgrade():
+    op.create_table(
+        "schema_versions",
+        sa.Column("game", sa.String(4), primary_key=True, nullable=False),
+        sa.Column("version", sa.Integer, nullable=False, server_default="1"),
+        mysql_charset="utf8mb4",
+    )

core/data/alembic/versions/e4e8d89c9b02_idac_battle_gift_and_tips_added.py

@@ -0,0 +1,83 @@
+"""IDAC Battle Gift and Tips added
+
+Revision ID: e4e8d89c9b02
+Revises: 81e44dd6047a
+Create Date: 2024-04-01 17:49:50.009718
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "e4e8d89c9b02"
+down_revision = "81e44dd6047a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "idac_user_battle_gift",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user", sa.Integer(), nullable=False),
+        sa.Column("battle_gift_event_id", sa.Integer(), nullable=True),
+        sa.Column("gift_id", sa.Integer(), nullable=True),
+        sa.Column("gift_status", sa.Integer(), nullable=True),
+        sa.Column(
+            "received_date",
+            sa.TIMESTAMP(),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user"], ["aime_user.id"], onupdate="cascade", ondelete="cascade"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "user", "battle_gift_event_id", "gift_id", name="idac_user_battle_gift_uk"
+        ),
+        mysql_charset="utf8mb4",
+    )
+
+    op.create_table(
+        "idac_profile_tips",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user", sa.Integer(), nullable=False),
+        sa.Column("version", sa.Integer(), nullable=False),
+        sa.Column(
+            "tips_list",
+            sa.String(length=16),
+            server_default="QAAAAAAAAAAAAAAA",
+            nullable=True,
+        ),
+        sa.Column(
+            "timetrial_play_count", sa.Integer(), server_default="0", nullable=True
+        ),
+        sa.Column("story_play_count", sa.Integer(), server_default="0", nullable=True),
+        sa.Column(
+            "store_battle_play_count", sa.Integer(), server_default="0", nullable=True
+        ),
+        sa.Column(
+            "online_battle_play_count", sa.Integer(), server_default="0", nullable=True
+        ),
+        sa.Column(
+            "special_play_count", sa.Integer(), server_default="0", nullable=True
+        ),
+        sa.Column(
+            "challenge_play_count", sa.Integer(), server_default="0", nullable=True
+        ),
+        sa.Column("theory_play_count", sa.Integer(), server_default="0", nullable=True),
+        sa.ForeignKeyConstraint(
+            ["user"], ["aime_user.id"], onupdate="cascade", ondelete="cascade"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("user", "version", name="idac_profile_tips_uk"),
+        mysql_charset="utf8mb4",
+    )
+
+
+def downgrade():
+    op.drop_table("idac_user_battle_gift")
+    op.drop_table("idac_profile_tips")

core/data/database.py

@@ -1,13 +1,14 @@
 import logging, coloredlogs
-from typing import Optional, Dict, List
+from typing import Optional
 from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
 from logging.handlers import TimedRotatingFileHandler
-import importlib, os
+import os
 import secrets, string
 import bcrypt
 from hashlib import sha256
+import alembic.config
+import glob
 
 from core.config import CoreConfig
 from core.data.schema import *
@@ -15,7 +16,6 @@ from core.utils import Utils
 
 
 class Data:
-    current_schema_version = 6
     engine = None
     session = None
     user = None
@@ -77,281 +77,177 @@ class Data:
             )
             self.logger.handler_set = True  # type: ignore
 
+    def __alembic_cmd(self, command: str, *args: str) -> None:
+        old_dir = os.path.abspath(os.path.curdir)
+        base_dir = os.path.join(os.path.abspath(os.path.curdir), 'core', 'data', 'alembic')
+        alembicArgs = [
+            "-c",
+            os.path.join(base_dir, "alembic.ini"),
+            "-x",
+            f"script_location={base_dir}",
+            "-x",
+            f"sqlalchemy.url={self.__url}",
+            command,
+        ]
+        alembicArgs.extend(args)
+        os.chdir(base_dir)
+        alembic.config.main(argv=alembicArgs)
+        os.chdir(old_dir)
+
     def create_database(self):
         self.logger.info("Creating databases...")
-        try:
-            metadata.create_all(self.__engine.connect())
-        except SQLAlchemyError as e:
-            self.logger.error(f"Failed to create databases! {e}")
-            return
-
-        games = Utils.get_all_titles()
-        for game_dir, game_mod in games.items():
-            try:
-                if hasattr(game_mod, "database") and hasattr(
-                    game_mod, "current_schema_version"
-                ):
-                    game_mod.database(self.config)
-                metadata.create_all(self.__engine.connect())
-
-                self.base.touch_schema_ver(
-                    game_mod.current_schema_version, game_mod.game_codes[0]
-                )
-
-            except Exception as e:
-                self.logger.warning(
-                    f"Could not load database schema from {game_dir} - {e}"
-                )
-
-        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.current_schema_version)
-
-        self.logger.info(
-            f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
-        )
-        self.user.reset_autoincrement(
-            self.config.database.user_table_autoincrement_start
+        metadata.create_all(
+            self.engine,
+            checkfirst=True,
         )
 
-    def recreate_database(self):
-        self.logger.info("Dropping all databases...")
-        self.base.execute("SET FOREIGN_KEY_CHECKS=0")
-        try:
-            metadata.drop_all(self.__engine.connect())
-        except SQLAlchemyError as e:
-            self.logger.error(f"Failed to drop databases! {e}")
-            return
+        for _, mod in Utils.get_all_titles().items():
+            if hasattr(mod, "database"):
+                mod.database(self.config)
+                metadata.create_all(
+                    self.engine,
+                    checkfirst=True,
+                )
 
-        for root, dirs, files in os.walk("./titles"):
-            for dir in dirs:
-                if not dir.startswith("__"):
-                    try:
-                        mod = importlib.import_module(f"titles.{dir}")
+        # Stamp the end revision as if alembic had created it, so it can take off after this.
+        self.__alembic_cmd(
+            "stamp",
+            "head",
+        )
 
-                        try:
-                            if hasattr(mod, "database"):
-                                mod.database(self.config)
-                            metadata.drop_all(self.__engine.connect())
+    def schema_upgrade(self, ver: str = None):
+        self.__alembic_cmd(
+            "upgrade",
+            "head" if not ver else ver,
+        )
 
-                        except Exception as e:
-                            self.logger.warning(
-                                f"Could not load database schema from {dir} - {e}"
-                            )
+    def schema_downgrade(self, ver: str):
+        self.__alembic_cmd(
+            "downgrade",
+            ver,
+        )
 
-                    except ImportError as e:
-                        self.logger.warning(
-                            f"Failed to load database schema dir {dir} - {e}"
-                        )
-            break
-
-        self.base.execute("SET FOREIGN_KEY_CHECKS=1")
-
-        self.create_database()
-
-    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
-        old_ver = self.base.get_schema_ver(game)
-        sql = ""
-        if version is None:
-            if not game == "CORE":
-                titles = Utils.get_all_titles()
-
-                for folder, mod in titles.items():
-                    if not mod.game_codes[0] == game:
-                        continue
-
-                    if hasattr(mod, "current_schema_version"):
-                        version = mod.current_schema_version
-
-                    else:
-                        self.logger.warning(
-                            f"current_schema_version not found for {folder}"
-                        )
-
-            else:
-                version = self.current_schema_version
-
-        if version is None:
-            self.logger.warning(
-                f"Could not determine latest version for {game}, please specify --version"
-            )
-
-        if old_ver is None:
-            self.logger.error(
-                f"Schema for game {game} does not exist, did you run the creation script?"
-            )
-            return
-
-        if old_ver == version:
-            self.logger.info(
-                f"Schema for game {game} is already version {old_ver}, nothing to do"
-            )
-            return
-
-        if action == "upgrade":
-            for x in range(old_ver, version):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        else:
-            for x in range(old_ver, version, -1):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        result = self.base.set_schema_ver(version, game)
-        if result is None:
-            self.logger.error("Error setting version in schema_version table!")
-            return None
-
-        self.logger.info(f"Successfully migrated {game} to schema version {version}")
-
-    def create_owner(self, email: Optional[str] = None) -> None:
+    async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None:
         pw = "".join(
             secrets.choice(string.ascii_letters + string.digits) for i in range(20)
         )
         hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
 
-        user_id = self.user.create_user(email=email, permission=255, password=hash)
+        user_id = await self.user.create_user(username="sysowner", email=email, password=hash.decode(), permission=255)
         if user_id is None:
             self.logger.error(f"Failed to create owner with email {email}")
             return
 
-        card_id = self.card.create_card(user_id, "00000000000000000000")
+        card_id = await self.card.create_card(user_id, code)
         if card_id is None:
             self.logger.error(f"Failed to create card for owner with id {user_id}")
             return
 
         self.logger.warning(
-            f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
+            f"Successfully created owner with email {email}, access code {code}, and password {pw} Make sure to change this password and assign a real card ASAP!"
         )
-
-    def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
-        if old_ac == new_ac:
-            self.logger.error("Both access codes are the same!")
-            return
-
-        new_card = self.card.get_card_by_access_code(new_ac)
-        if new_card is None:
-            self.card.update_access_code(old_ac, new_ac)
-            return
-
-        if not should_force:
-            self.logger.warning(
-                f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
-                f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
-            )
-            return
-
-        self.logger.info(
-            f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
-        )
-        self.card.delete_card(new_card["id"])
-        self.card.update_access_code(old_ac, new_ac)
-
-        hanging_user = self.user.get_user(new_card["user"])
-        if hanging_user["password"] is None:
-            self.logger.info(f"Delete hanging user {hanging_user['id']}")
-            self.user.delete_user(hanging_user["id"])
-
-    def delete_hanging_users(self) -> None:
-        """
-        Finds and deletes users that have not registered for the webui that have no cards assocated with them.
-        """
-        unreg_users = self.user.get_unregistered_users()
-        if unreg_users is None:
-            self.logger.error("Error occoured finding unregistered users")
-
-        for user in unreg_users:
-            cards = self.card.get_user_cards(user["id"])
-            if cards is None:
-                self.logger.error(f"Error getting cards for user {user['id']}")
-                continue
-
-            if not cards:
-                self.logger.info(f"Delete hanging user {user['id']}")
-                self.user.delete_user(user["id"])
-
-    def autoupgrade(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
-        if all_game_versions is None:
-            self.logger.warning("Failed to get schema versions")
-            return
-
-        all_games = Utils.get_all_titles()
-        all_games_list: Dict[str, int] = {}
-        for _, mod in all_games.items():
-            if hasattr(mod, "current_schema_version"):
-                all_games_list[mod.game_codes[0]] = mod.current_schema_version
-
-        for x in all_game_versions:
-            failed = False
-            game = x["game"].upper()
-            update_ver = int(x["version"])
-            latest_ver = all_games_list.get(game, 1)
-            if game == "CORE":
-                latest_ver = self.current_schema_version
-
-            if update_ver == latest_ver:
-                self.logger.info(f"{game} is already latest version")
-                continue
-
-            for y in range(update_ver + 1, latest_ver + 1):
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
-                    with open(
-                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
-                        "r",
-                        encoding="utf-8",
-                    ) as f:
-                        sql = f.read()
-
-                    result = self.base.execute(sql)
-                    if result is None:
-                        self.logger.error(
-                            f"Error execuing sql script for game {game} v{y}!"
-                        )
-                        failed = True
-                        break
-                else:
-                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
-                    failed = True
-
-            if not failed:
-                self.base.set_schema_ver(latest_ver, game)
     
-    def show_versions(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
-        for ver in all_game_versions:
-            self.logger.info(f"{ver['game']} -> v{ver['version']}")
+    async def migrate(self) -> None:
+        exist = await self.base.execute("SELECT * FROM alembic_version")
+        if exist is not None:
+            self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!")
+            return
+        
+        self.logger.info("Upgrading to latest with legacy system")
+        if not await self.legacy_upgrade():
+            self.logger.warn("No need to migrate as you have already deleted the old schema_versions system. If you are trying to upgrade the schema, use `upgrade` instead!")
+            return
+        self.logger.info("Done")
+        
+        self.logger.info("Stamp with initial revision")
+        self.__alembic_cmd(
+            "stamp",
+            "835b862f9bf0",
+        )
+
+        self.logger.info("Upgrade")
+        self.__alembic_cmd(
+            "upgrade",
+            "head",
+        )
+    
+    async def legacy_upgrade(self) -> bool:
+        vers = await self.base.execute("SELECT * FROM schema_versions")
+        if vers is None:
+            self.logger.warn("Cannot legacy upgrade, schema_versions table unavailable!")
+            return False
+        
+        db_vers = {}
+        vers_list = vers.fetchall()
+        for x in vers_list:
+            db_vers[x['game']] = x['version']
+        
+        core_now_ver = int(db_vers['CORE']) + 1
+        while os.path.exists(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql"):
+            with open(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql", "r") as f:
+                result = await self.base.execute(f.read())
+                
+                if result is None:
+                    self.logger.error(f"Invalid upgrade script CORE_{core_now_ver}_upgrade.sql")
+                    break
+
+                result = await self.base.execute(f"UPDATE schema_versions SET version = {core_now_ver} WHERE game = 'CORE'")
+                if result is None:
+                    self.logger.error(f"Failed to update schema version for CORE to {core_now_ver}")
+                    break
+            
+            self.logger.info(f"Upgrade CORE to version {core_now_ver}")
+            core_now_ver += 1
+        
+        for _, mod in Utils.get_all_titles().items():
+            game_codes = getattr(mod, "game_codes", [])
+            for game in game_codes:
+                if game not in db_vers:
+                    self.logger.warn(f"{game} does not have an antry in schema_versions, skipping")
+                    continue
+
+                now_ver = int(db_vers[game]) + 1
+                while os.path.exists(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql"):
+                    with open(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql", "r") as f:
+                        result = await self.base.execute(f.read())
+                        
+                        if result is None:
+                            self.logger.error(f"Invalid upgrade script {game}_{now_ver}_upgrade.sql")
+                            break
+
+                        result = await self.base.execute(f"UPDATE schema_versions SET version = {now_ver} WHERE game = '{game}'")
+                        if result is None:
+                            self.logger.error(f"Failed to update schema version for {game} to {now_ver}")
+                            break
+
+                    self.logger.info(f"Upgrade {game} to version {now_ver}")
+                    now_ver += 1
+        
+        return True
+
+    async def create_revision(self, message: str) -> None:
+        if not message:
+            self.logger.info("Message is required for create-revision")
+            return
+        
+        self.__alembic_cmd(
+            "revision",
+            "-m",
+            message,
+        )
+
+    async def create_revision_auto(self, message: str) -> None:
+        if not message:
+            self.logger.info("Message is required for create-revision")
+            return
+        
+        for _, mod in Utils.get_all_titles().items():
+            if hasattr(mod, "database"):
+                mod.database(self.config)
+        
+        self.__alembic_cmd(
+            "revision",
+            "--autogenerate",
+            "-m",
+            message,
+        )

core/data/schema/arcade.py

@@ -69,7 +69,7 @@ arcade_owner = Table(
 
 
 class ArcadeData(BaseData):
-    def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
+    async def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
         if serial is not None:
             serial = serial.replace("-", "")
             if len(serial) == 11:
@@ -89,12 +89,12 @@ class ArcadeData(BaseData):
             self.logger.error(f"{__name__ }: Need either serial or ID to look up!")
             return None
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_machine(
+    async def create_machine(
         self,
         arcade_id: int,
         serial: str = "",
@@ -102,21 +102,21 @@ class ArcadeData(BaseData):
         game: str = None,
         is_cab: bool = False,
     ) -> Optional[int]:
-        if arcade_id:
+        if not arcade_id:
             self.logger.error(f"{__name__ }: Need arcade id!")
             return None
 
         sql = machine.insert().values(
-            arcade=arcade_id, keychip=serial, board=board, game=game, is_cab=is_cab
+            arcade=arcade_id, serial=serial, board=board, game=game, is_cab=is_cab
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def set_machine_serial(self, machine_id: int, serial: str) -> None:
-        result = self.execute(
+    async def set_machine_serial(self, machine_id: int, serial: str) -> None:
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(keychip=serial)
         )
         if result is None:
@@ -125,8 +125,8 @@ class ArcadeData(BaseData):
             )
         return result.lastrowid
 
-    def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
-        result = self.execute(
+    async def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(board=boardid)
         )
         if result is None:
@@ -134,29 +134,29 @@ class ArcadeData(BaseData):
                 f"Failed to update board id for machine {machine_id} -> {boardid}"
             )
 
-    def get_arcade(self, id: int) -> Optional[Row]:
+    async def get_arcade(self, id: int) -> Optional[Row]:
         sql = arcade.select(arcade.c.id == id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
     
-    def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
+    async def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
         sql = machine.select(machine.c.arcade == id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_arcade(
+    async def create_arcade(
         self,
-        name: str,
+        name: str = None,
         nickname: str = None,
         country: str = "JPN",
         country_id: int = 1,
         state: str = "",
         city: str = "",
-        regional_id: int = 1,
+        region_id: int = 1,
     ) -> Optional[int]:
         if nickname is None:
             nickname = name
@@ -168,46 +168,46 @@ class ArcadeData(BaseData):
             country_id=country_id,
             state=state,
             city=city,
-            regional_id=regional_id,
+            region_id=region_id,
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
+    async def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
         sql = select(arcade).join(arcade_owner, arcade_owner.c.arcade == arcade.c.id).where(arcade_owner.c.user == user_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchall()
     
-    def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
+    async def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
         sql = select(arcade_owner.c.permissions).where(and_(arcade_owner.c.user == user_id, arcade_owner.c.arcade == arcade_id))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
+    async def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
         sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
+    async def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
         sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def format_serial(
-        self, platform_code: str, platform_rev: int, serial_num: int, append: int = 4152
+    def format_serial( # TODO: Actual serial stuff
+        self, platform_code: str, platform_rev: int, serial_num: int, append: int = 8888
     ) -> str:
         return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
 
@@ -217,16 +217,16 @@ class ArcadeData(BaseData):
         
         return True
 
-    def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
+    async def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
         sql = arcade.select(or_(arcade.c.name.like(f"%{name}%"), arcade.c.nickname.like(f"%{name}%")))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
+    async def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
         sql = arcade.select().where(arcade.c.ip == ip)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()

core/data/schema/base.py

@@ -15,14 +15,6 @@ from core.config import CoreConfig
 
 metadata = MetaData()
 
-schema_ver = Table(
-    "schema_versions",
-    metadata,
-    Column("game", String(4), primary_key=True, nullable=False),
-    Column("version", Integer, nullable=False, server_default="1"),
-    mysql_charset="utf8mb4",
-)
-
 event_log = Table(
     "event_log",
     metadata,
@@ -43,11 +35,11 @@ class BaseData:
         self.conn = conn
         self.logger = logging.getLogger("database")
 
-    def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
+    async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
+            self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:
@@ -82,52 +74,7 @@ class BaseData:
         """
         return randrange(10000, 9999999)
 
-    def get_all_schema_vers(self) -> Optional[List[Row]]:
-        sql = select(schema_ver)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-        return result.fetchall()
-
-    def get_schema_ver(self, game: str) -> Optional[int]:
-        sql = select(schema_ver).where(schema_ver.c.game == game)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-
-        row = result.fetchone()
-        if row is None:
-            return None
-
-        return row["version"]
-    
-    def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=ver)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def log_event(
+    async def log_event(
         self, system: str, type: str, severity: int, message: str, details: Dict = {}
     ) -> Optional[int]:
         sql = event_log.insert().values(
@@ -137,7 +84,7 @@ class BaseData:
             message=message,
             details=json.dumps(details),
         )
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             self.logger.error(
@@ -147,9 +94,9 @@ class BaseData:
 
         return result.lastrowid
 
-    def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
+    async def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
         sql = event_log.select().limit(entries).all()
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             return None

core/data/schema/card.py

@@ -27,91 +27,101 @@ aime_card = Table(
 
 
 class CardData(BaseData):
-    def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
+    async def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.access_code == access_code)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_card_by_id(self, card_id: int) -> Optional[Row]:
+    async def get_card_by_id(self, card_id: int) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def update_access_code(self, old_ac: str, new_ac: str) -> None:
+    async def update_access_code(self, old_ac: str, new_ac: str) -> None:
         sql = aime_card.update(aime_card.c.access_code == old_ac).values(
             access_code=new_ac
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(
                 f"Failed to change card access code from {old_ac} to {new_ac}"
             )
 
-    def get_user_id_from_card(self, access_code: str) -> Optional[int]:
+    async def get_user_id_from_card(self, access_code: str) -> Optional[int]:
         """
         Given a 20 digit access code as a string, get the user id associated with that card
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
 
         return int(card["user"])
 
-    def get_card_banned(self, access_code: str) -> Optional[bool]:
+    async def get_card_banned(self, access_code: str) -> Optional[bool]:
         """
         Given a 20 digit access code as a string, check if the card is banned
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
         if card["is_banned"]:
             return True
         return False
-    def get_card_locked(self, access_code: str) -> Optional[bool]:
+    
+    async def get_card_locked(self, access_code: str) -> Optional[bool]:
         """
         Given a 20 digit access code as a string, check if the card is locked
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
         if card["is_locked"]:
             return True
         return False
 
-    def delete_card(self, card_id: int) -> None:
+    async def delete_card(self, card_id: int) -> None:
         sql = aime_card.delete(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete card with id {card_id}")
 
-    def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
         """
         Returns all cards owned by a user
         """
         sql = aime_card.select(aime_card.c.user == aime_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def create_card(self, user_id: int, access_code: str) -> Optional[int]:
+    async def create_card(self, user_id: int, access_code: str) -> Optional[int]:
         """
         Given a aime_user id and a 20 digit access code as a string, create a card and return the ID if successful
         """
         sql = aime_card.insert().values(user=user_id, access_code=access_code)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
+    async def update_card_last_login(self, access_code: str) -> None:
+        sql = aime_card.update(aime_card.c.access_code == access_code).values(
+            last_login_date=func.now()
+        )
+        
+        result = await self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to update last login time for {access_code}")
+    
     def to_access_code(self, luid: str) -> str:
         """
         Given a felica cards internal 16 hex character luid, convert it to a 0-padded 20 digit access code as a string

core/data/schema/user.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Optional, List
 from sqlalchemy import Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP
@@ -24,15 +23,8 @@ aime_user = Table(
     mysql_charset="utf8mb4",
 )
 
-
-class PermissionBits(Enum):
-    PermUser = 1
-    PermMod = 2
-    PermSysAdmin = 4
-
-
 class UserData(BaseData):
-    def create_user(
+    async def create_user(
         self,
         id: int = None,
         username: str = None,
@@ -60,20 +52,20 @@ class UserData(BaseData):
             username=username, email=email, password=password, permissions=permission
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_user(self, user_id: int) -> Optional[Row]:
+    async def get_user(self, user_id: int) -> Optional[Row]:
         sql = select(aime_user).where(aime_user.c.id == user_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def check_password(self, user_id: int, passwd: bytes = None) -> bool:
-        usr = self.get_user(user_id)
+    async def check_password(self, user_id: int, passwd: bytes = None) -> bool:
+        usr = await self.get_user(user_id)
         if usr is None:
             return False
 
@@ -85,39 +77,46 @@ class UserData(BaseData):
 
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
-    def reset_autoincrement(self, ai_value: int) -> None:
-        # ALTER TABLE isn't in sqlalchemy so we do this the ugly way
-        sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
-        self.execute(sql)
-
-    def delete_user(self, user_id: int) -> None:
+    async def delete_user(self, user_id: int) -> None:
         sql = aime_user.delete(aime_user.c.id == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete user with id {user_id}")
 
-    def get_unregistered_users(self) -> List[Row]:
+    async def get_unregistered_users(self) -> List[Row]:
         """
         Returns a list of users who have not registered with the webui. They may or may not have cards.
         """
         sql = select(aime_user).where(aime_user.c.password == None)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def find_user_by_email(self, email: str) -> Row:
+    async def find_user_by_email(self, email: str) -> Row:
         sql = select(aime_user).where(aime_user.c.email == email)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def find_user_by_username(self, username: str) -> List[Row]:
+    async def find_user_by_username(self, username: str) -> List[Row]:
         sql = aime_user.select(aime_user.c.username.like(f"%{username}%"))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchall()
+
+    async def change_password(self, user_id: int, new_passwd: str) -> bool:
+        sql = aime_user.update(aime_user.c.id == user_id).values(password = new_passwd)
+
+        result = await self.execute(sql)
+        return result is not None
+
+    async def change_username(self, user_id: int, new_name: str) -> bool:
+        sql = aime_user.update(aime_user.c.id == user_id).values(username = new_name)
+
+        result = await self.execute(sql)
+        return result is not None

core/data/schema/versions/SDDT_5_rollback.sql

@@ -1,8 +1,8 @@
 SET FOREIGN_KEY_CHECKS=0;
 
 ALTER TABLE ongeki_user_event_point DROP COLUMN version;
-ALTER TABLE ongeki_user_event_point DROP COLUMN rank;
-ALTER TABLE ongeki_user_event_point DROP COLUMN type;
+ALTER TABLE ongeki_user_event_point DROP COLUMN `rank`;
+ALTER TABLE ongeki_user_event_point DROP COLUMN `type`;
 ALTER TABLE ongeki_user_event_point DROP COLUMN date;
 
 ALTER TABLE ongeki_user_tech_event DROP COLUMN version;
@@ -19,4 +19,4 @@ DROP TABLE ongeki_static_tech_music;
 DROP TABLE ongeki_static_client_testmode;
 DROP TABLE ongeki_static_game_point;
 
-SET FOREIGN_KEY_CHECKS=1;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDDT_6_upgrade.sql

@@ -1,8 +1,8 @@
 SET FOREIGN_KEY_CHECKS=0;
 
 ALTER TABLE ongeki_user_event_point ADD COLUMN version INTEGER NOT NULL;
-ALTER TABLE ongeki_user_event_point ADD COLUMN rank INTEGER;
-ALTER TABLE ongeki_user_event_point ADD COLUMN type INTEGER NOT NULL;
+ALTER TABLE ongeki_user_event_point ADD COLUMN `rank` INTEGER;
+ALTER TABLE ongeki_user_event_point ADD COLUMN `type` INTEGER NOT NULL;
 ALTER TABLE ongeki_user_event_point ADD COLUMN date VARCHAR(25);
 
 ALTER TABLE ongeki_user_tech_event ADD COLUMN version INTEGER NOT NULL;
@@ -12,87 +12,87 @@ ALTER TABLE ongeki_user_mission_point ADD COLUMN version INTEGER NOT NULL;
 ALTER TABLE ongeki_static_events ADD COLUMN endDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
 
 CREATE TABLE ongeki_tech_event_ranking (
-	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	user INT NOT NULL,
-	version INT NOT NULL,
-	date VARCHAR(25),
-	eventId INT NOT NULL,
-	rank INT,
-	totalPlatinumScore INT NOT NULL,
-	totalTechScore INT NOT NULL,
-	UNIQUE KEY ongeki_tech_event_ranking_uk (user, eventId),
-	CONSTRAINT ongeki_tech_event_ranking_ibfk1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE ON UPDATE CASCADE
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    user INT NOT NULL,
+    version INT NOT NULL,
+    date VARCHAR(25),
+    eventId INT NOT NULL,
+    `rank` INT,
+    totalPlatinumScore INT NOT NULL,
+    totalTechScore INT NOT NULL,
+    UNIQUE KEY ongeki_tech_event_ranking_uk (user, eventId),
+    CONSTRAINT ongeki_tech_event_ranking_ibfk1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE ON UPDATE CASCADE
 );
 
 CREATE TABLE ongeki_static_music_ranking_list (
-	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	version INT NOT NULL,
-	musicId INT NOT NULL,
-	point INT NOT NULL,
-	userName VARCHAR(255),
-	UNIQUE KEY ongeki_static_music_ranking_list_uk (version, musicId)
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    musicId INT NOT NULL,
+    point INT NOT NULL,
+    userName VARCHAR(255),
+    UNIQUE KEY ongeki_static_music_ranking_list_uk (version, musicId)
 );
 
 CREATE TABLE ongeki_static_rewards (
-	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	version INT NOT NULL,
-	rewardId INT NOT NULL,
-	rewardName VARCHAR(255) NOT NULL,
-	itemKind INT NOT NULL,
-	itemId INT NOT NULL,
-	UNIQUE KEY ongeki_tech_event_ranking_uk (version, rewardId)
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    rewardId INT NOT NULL,
+    rewardName VARCHAR(255) NOT NULL,
+    itemKind INT NOT NULL,
+    itemId INT NOT NULL,
+    UNIQUE KEY ongeki_tech_event_ranking_uk (version, rewardId)
 );
 
 CREATE TABLE ongeki_static_present_list (
-	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	version INT NOT NULL,
-	presentId INT NOT NULL,
-	presentName VARCHAR(255) NOT NULL,
-	rewardId INT NOT NULL,
-	stock INT NOT NULL,
-	message VARCHAR(255),
-	startDate VARCHAR(25) NOT NULL,
-	endDate VARCHAR(25) NOT NULL,
-	UNIQUE KEY ongeki_static_present_list_uk (version, presentId, rewardId)
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    presentId INT NOT NULL,
+    presentName VARCHAR(255) NOT NULL,
+    rewardId INT NOT NULL,
+    stock INT NOT NULL,
+    message VARCHAR(255),
+    startDate VARCHAR(25) NOT NULL,
+    endDate VARCHAR(25) NOT NULL,
+    UNIQUE KEY ongeki_static_present_list_uk (version, presentId, rewardId)
 );
 
 CREATE TABLE ongeki_static_tech_music (
-	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	version INT NOT NULL,
-	eventId INT NOT NULL,
-	musicId INT NOT NULL,
-	level INT NOT NULL,
-	UNIQUE KEY ongeki_static_tech_music_uk (version, musicId, eventId)
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    version INT NOT NULL,
+    eventId INT NOT NULL,
+    musicId INT NOT NULL,
+    level INT NOT NULL,
+    UNIQUE KEY ongeki_static_tech_music_uk (version, musicId, eventId)
 );
 
 CREATE TABLE ongeki_static_client_testmode (
-	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	regionId INT NOT NULL,
-	placeId INT NOT NULL,
-	clientId VARCHAR(11) NOT NULL,
-	updateDate TIMESTAMP NOT NULL,
-	isDelivery BOOLEAN NOT NULL,
-	groupId INT NOT NULL,
-	groupRole INT NOT NULL,
-	continueMode INT NOT NULL,
-	selectMusicTime INT NOT NULL,
-	advertiseVolume INT NOT NULL,
-	eventMode INT NOT NULL,
-	eventMusicNum INT NOT NULL,
-	patternGp INT NOT NULL,
-	limitGp INT NOT NULL,
-	maxLeverMovable INT NOT NULL,
-	minLeverMovable INT NOT NULL,
-	UNIQUE KEY ongeki_static_client_testmode_uk (clientId)
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    regionId INT NOT NULL,
+    placeId INT NOT NULL,
+    clientId VARCHAR(11) NOT NULL,
+    updateDate TIMESTAMP NOT NULL,
+    isDelivery BOOLEAN NOT NULL,
+    groupId INT NOT NULL,
+    groupRole INT NOT NULL,
+    continueMode INT NOT NULL,
+    selectMusicTime INT NOT NULL,
+    advertiseVolume INT NOT NULL,
+    eventMode INT NOT NULL,
+    eventMusicNum INT NOT NULL,
+    patternGp INT NOT NULL,
+    limitGp INT NOT NULL,
+    maxLeverMovable INT NOT NULL,
+    minLeverMovable INT NOT NULL,
+    UNIQUE KEY ongeki_static_client_testmode_uk (clientId)
 );
 
 CREATE TABLE ongeki_static_game_point (
-	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	type INT NOT NULL,
-	cost INT NOT NULL,
-	startDate VARCHAR(25) NOT NULL DEFAULT "2000-01-01 05:00:00.0",
-	endDate VARCHAR(25) NOT NULL DEFAULT "2099-01-01 05:00:00.0",
-	UNIQUE KEY ongeki_static_game_point_uk (type)
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    `type` INT NOT NULL,
+    cost INT NOT NULL,
+    startDate VARCHAR(25) NOT NULL DEFAULT "2000-01-01 05:00:00.0",
+    endDate VARCHAR(25) NOT NULL DEFAULT "2099-01-01 05:00:00.0",
+    UNIQUE KEY ongeki_static_game_point_uk (`type`)
 );
 
-SET FOREIGN_KEY_CHECKS=1;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDGT_1_rollback.sql

@@ -0,0 +1,54 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- WARNING: This script is NOT idempotent! MAKE A BACKUP BEFORE RUNNING THIS SCRIPT!
+
+-- Drop UK idac_user_vs_info_uk
+ALTER TABLE idac_user_vs_info
+DROP FOREIGN KEY idac_user_vs_info_ibfk_1,
+DROP INDEX idac_user_vs_info_uk;
+
+-- Drop the new columns added to the original table
+ALTER TABLE idac_user_vs_info
+DROP COLUMN battle_mode,
+DROP COLUMN invalid,
+DROP COLUMN str,
+DROP COLUMN str_now,
+DROP COLUMN lose_now;
+
+-- Add back the old columns to the original table
+ALTER TABLE idac_user_vs_info
+ADD COLUMN group_key VARCHAR(25),
+ADD COLUMN win_flg INT,
+ADD COLUMN style_car_id INT,
+ADD COLUMN course_id INT,
+ADD COLUMN course_day INT,
+ADD COLUMN players_num INT,
+ADD COLUMN winning INT,
+ADD COLUMN advantage_1 INT,
+ADD COLUMN advantage_2 INT,
+ADD COLUMN advantage_3 INT,
+ADD COLUMN advantage_4 INT,
+ADD COLUMN select_course_id INT,
+ADD COLUMN select_course_day INT,
+ADD COLUMN select_course_random INT,
+ADD COLUMN matching_success_sec INT,
+ADD COLUMN boost_flag INT;
+
+-- Delete the data from the original table where group_key is NULL
+DELETE FROM idac_user_vs_info
+WHERE group_key IS NULL;
+
+-- Insert data back to the original table from idac_user_vs_course_info
+INSERT INTO idac_user_vs_info (user, group_key, win_flg, style_car_id, course_id, course_day, players_num, winning, advantage_1, advantage_2, advantage_3, advantage_4, select_course_id, select_course_day, select_course_random, matching_success_sec, boost_flag, vs_history, break_count, break_penalty_flag)
+SELECT user, CONCAT(FLOOR(RAND()*(99999999999999-10000000000000+1)+10000000000000), 'A69E01A8888'), 0, 0, course_id, 0, 0, vs_cnt, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+FROM idac_user_vs_course_info;
+
+-- Add back the constraints and indexes to the original table
+ALTER TABLE idac_user_vs_info
+ADD CONSTRAINT idac_user_vs_info_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE ON UPDATE CASCADE,
+ADD UNIQUE KEY idac_user_vs_info_uk (user, group_key);
+
+-- Drop the new table idac_user_vs_course_info
+DROP TABLE IF EXISTS idac_user_vs_course_info;
+
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDGT_2_upgrade.sql

@@ -0,0 +1,71 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- WARNING: This script is NOT idempotent! MAKE A BACKUP BEFORE RUNNING THIS SCRIPT!
+
+-- Create the new table idac_user_vs_course_info
+CREATE TABLE idac_user_vs_course_info (
+    id INT PRIMARY KEY AUTO_INCREMENT,
+    user INT,
+    battle_mode INT,
+    course_id INT,
+    vs_cnt INT,
+    vs_win INT,
+    CONSTRAINT idac_user_vs_course_info_fk FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE ON UPDATE CASCADE,
+    UNIQUE KEY idac_user_vs_course_info_uk (user, battle_mode, course_id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Insert data from the original table to the new tables
+INSERT INTO idac_user_vs_course_info (user, battle_mode, course_id, vs_cnt, vs_win)
+SELECT user, 1 as battle_mode, course_id, COUNT(winning) as vs_cnt, SUM(win_flg) as vs_win
+FROM idac_user_vs_info
+GROUP BY user, course_id;
+
+-- Drop UK idac_user_vs_info_uk
+ALTER TABLE idac_user_vs_info
+DROP FOREIGN KEY idac_user_vs_info_ibfk_1,
+DROP INDEX idac_user_vs_info_uk;
+
+-- Drop/Add the old columns from the original table
+ALTER TABLE idac_user_vs_info
+DROP COLUMN group_key,
+DROP COLUMN win_flg,
+DROP COLUMN style_car_id,
+DROP COLUMN course_id,
+DROP COLUMN course_day,
+DROP COLUMN players_num,
+DROP COLUMN winning,
+DROP COLUMN advantage_1,
+DROP COLUMN advantage_2,
+DROP COLUMN advantage_3,
+DROP COLUMN advantage_4,
+DROP COLUMN select_course_id,
+DROP COLUMN select_course_day,
+DROP COLUMN select_course_random,
+DROP COLUMN matching_success_sec,
+DROP COLUMN boost_flag,
+
+ADD COLUMN battle_mode TINYINT UNSIGNED DEFAULT 1 NOT NULL AFTER user,
+ADD COLUMN invalid INT DEFAULT 0,
+ADD COLUMN str INT DEFAULT 0,
+ADD COLUMN str_now INT DEFAULT 0,
+ADD COLUMN lose_now INT DEFAULT 0;
+
+-- Create a temporary table to store the records you want to keep
+CREATE TEMPORARY TABLE temp_table AS
+SELECT MIN(id) AS min_id
+FROM idac_user_vs_info
+GROUP BY battle_mode, user;
+
+-- Delete records from the original table based on the temporary table
+DELETE FROM idac_user_vs_info
+WHERE id NOT IN (SELECT min_id FROM temp_table);
+
+-- Drop the temporary table
+DROP TEMPORARY TABLE IF EXISTS temp_table;
+
+-- Add UK idac_user_vs_info_uk
+ALTER TABLE idac_user_vs_info
+ADD CONSTRAINT idac_user_vs_info_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE ON UPDATE CASCADE,
+ADD UNIQUE KEY idac_user_vs_info_uk (user, battle_mode);
+
+SET FOREIGN_KEY_CHECKS=1;

core/frontend/arcade/index.jinja

@@ -1,4 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>{{ arcade.name }}</h1>
-{% endblock content %}

core/frontend/machine/index.jinja

@@ -1,5 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-{% include "core/frontend/widgets/err_banner.jinja" %}
-<h1>Machine Management</h1>
-{% endblock content %}

core/frontend/sys/index.jinja

@@ -1,103 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>System Management</h1>
-
-<div class="row" id="rowForm">
-    {% if sesh.permissions >= 2 %}
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
-            <h3>User Search</h3>
-            <div class="form-group">
-                <label for="usrId">User ID</label>
-                <input type="number" class="form-control" id="usrId" name="usrId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="usrName">Username</label>
-                <input type="text" class="form-control" id="usrName" name="usrName">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="usrEmail">Email address</label>
-                <input type="email" class="form-control" id="usrEmail" name="usrEmail" aria-describedby="emailHelp">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    {% endif %}
-    {% if sesh.permissions >= 4 %}
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="arcadeLookup" name="arcadeLookup" action="/sys/lookup.arcade" class="form-inline" >
-            <h3>Arcade Search</h3>
-            <div class="form-group">
-                <label for="arcadeId">Arcade ID</label>
-                <input type="number" class="form-control" id="arcadeId" name="arcadeId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeName">Arcade Name</label>
-                <input type="text" class="form-control" id="arcadeName" name="arcadeName">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeUser">Owner User ID</label>
-                <input type="number" class="form-control" id="arcadeUser" name="arcadeUser">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeIp">Assigned IP Address</label>
-                <input type="text" class="form-control" id="arcadeIp" name="arcadeIp">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="cabLookup" name="cabLookup" action="/sys/lookup.cab" class="form-inline" >
-            <h3>Machine Search</h3>
-            <div class="form-group">
-                <label for="cabId">Machine ID</label>
-                <input type="number" class="form-control" id="cabId" name="cabId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="cabSerial">Machine Serial</label>
-                <input type="text" class="form-control" id="cabSerial" name="cabSerial">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="cabAcId">Arcade ID</label>
-                <input type="number" class="form-control" id="cabAcId" name="cabAcId">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    {% endif %}
-</div>
-<div class="row" id="rowResult" style="margin: 10px;">
-    {% if sesh.permissions >= 2 %}
-    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for usr in usrlist %}
-        <a href=/user/{{ usr.id }}><pre>{{ usr.id }} | {{ usr.username if usr.username != None else "<i>No Name Set</i>"}}</pre></a>
-        {% endfor %}
-    </div>
-    {% endif %}
-    {% if sesh.permissions >= 4 %}
-    <div id="arcadeSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for ac in aclist %}
-        <pre><a href=/arcade/{{ ac.id }}>{{ ac.id }} | {{ ac.name if ac.name != None else "<i>No Name Set</i>" }} | {{ ac.ip if ac.ip != None else "<i>No IP Assigned</i>"}}</pre></a>
-        {% endfor %}
-    </div
-    ><div id="cabSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for cab in cablist %}
-        <a href=/cab/{{ cab.id }}><pre>{{ cab.id }} | {{ cab.game if cab.game != None else "<i>ANY </i>" }} | {{ cab.serial }}</pre></a>
-        {% endfor %}
-    </div>
-    {% endif %}
-</div>
-<div class="row" id="rowAdd">
-
-</div>
-{% endblock content %}

core/frontend/user/index.jinja

@@ -1,41 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>Management for {{ username }}</h1>
-<h2>Cards <button class="btn btn-success" data-bs-toggle="modal" data-bs-target="#card_add">Add</button></h2>
-<ul style="font-size: 20px;">
-{% for c in cards %}
-<li>{{ c.access_code }}: {{ c.status }}&nbsp;{% if c.status == 'Active'%}<button class="btn-warning btn">Lock</button>{% elif c.status == 'Locked' %}<button class="btn-warning btn">Unlock</button>{% endif %}&nbsp;<button class="btn-danger btn">Delete</button></li>
-{% endfor %}
-</ul>
-
-{% if arcades is defined %}
-<h2>Arcades</h2>
-  <ul style="font-size: 20px;">
-  {% for a in arcades %}
-    <li><a href=/arcade/{{ a.id }}>{{ a.name }}</a></li>
-  {% endfor %}
-  </ul>
-{% endif %}
-
-<div class="modal fade" id="card_add" tabindex="-1" aria-labelledby="card_add_label" aria-hidden="true">
-  <div class="modal-dialog">
-    <div class="modal-content">
-      <div class="modal-header">
-        <h1 class="modal-title fs-5" id="card_add_label">Add Card</h1>
-        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
-      </div>
-      <div class="modal-body">
-        HOW TO:<br>
-        Scan your card on any networked game and press the "View Access Code" button (varies by game) and enter the 20 digit code below.<br>
-        !!FOR AMUSEIC CARDS: DO NOT ENTER THE CODE SHOWN ON THE BACK OF THE CARD ITSELF OR IT WILL NOT WORK!!
-        <p /><label for="card_add_frm_access_code">Access Code:&nbsp;</label><input id="card_add_frm_access_code" maxlength="20" type="text" required> 
-      </div>
-      <div class="modal-footer">
-        <button type="button" class="btn btn-primary">Add</button>
-        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
-      </div>
-    </div>
-  </div>
-</div>
-
-{% endblock content %}

core/frontend/widgets/err_banner.jinja

@@ -1,18 +0,0 @@
-{% if error > 0 %}
-<div class="err-banner">
-<h3>Error</h3>
-{% if error == 1 %}
-Card not registered, or wrong password
-{% elif error == 2 %}
-Missing or malformed access code
-{% elif error == 3 %}
-Failed to create user
-{% elif error == 4 %}
-Arcade not found
-{% elif error == 5 %}
-Machine not found
-{% else %}
-An unknown error occoured
-{% endif %}
-</div>
-{% endif %}

core/mucha.py

@@ -1,8 +1,8 @@
-from typing import Dict, Any, Optional, List
+from typing import Dict, Any, Optional
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web import resource
-from twisted.web.http import Request
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
 from datetime import datetime
 from Crypto.Cipher import Blowfish
 import pytz
@@ -10,9 +10,11 @@ import pytz
 from .config import CoreConfig
 from .utils import Utils
 from .title import TitleServlet
+from .data import Data
+from .const import *
 
 class MuchaServlet:
-    mucha_registry: List[str] = []
+    mucha_registry: Dict[str, Dict[str, str]] = {}
     def __init__(self, cfg: CoreConfig, cfg_dir: str) -> None:
         self.config = cfg
         self.config_dir = cfg_dir
@@ -36,90 +38,148 @@ class MuchaServlet:
 
         self.logger.setLevel(cfg.mucha.loglevel)
         coloredlogs.install(level=cfg.mucha.loglevel, logger=self.logger, fmt=log_fmt_str)
+        
+        self.data = Data(cfg)
 
         for _, mod in TitleServlet.title_registry.items():
-            if hasattr(mod, "get_mucha_info"):
-                enabled, game_cd = mod.get_mucha_info(
-                    self.config, self.config_dir
-                )
-                if enabled:
-                    self.mucha_registry.append(game_cd)
+            enabled, game_cds, netids = mod.get_mucha_info(self.config, self.config_dir)
+            if enabled:
+                for x in range(len(game_cds)):
+                    self.mucha_registry[game_cds[x]] = { "netid_prefix": netids[x] }
 
         self.logger.info(f"Serving {len(self.mucha_registry)} games")
 
-    def handle_boardauth(self, request: Request, _: Dict) -> bytes:
-        req_dict = self.mucha_preprocess(request.content.getvalue())
+    async def handle_boardauth(self, request: Request) -> bytes:
+        bod = await request.body()
+        req_dict = self.mucha_preprocess(bod)
         client_ip = Utils.get_ip_addr(request)
 
         if req_dict is None:
             self.logger.error(
-                f"Error processing mucha request {request.content.getvalue()}"
+                f"Error processing mucha request {bod}"
             )
-            return b"RESULTS=000"
+            return PlainTextResponse("RESULTS=000")
 
         req = MuchaAuthRequest(req_dict)
-        self.logger.info(f"Boardauth request from {client_ip} for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
+        
+        if not req.gameCd or not req.gameVer or not req.sendDate or not req.countryCd or not req.serialNum:
+            self.logger.warn(f"Missing required fields - {vars(req)}")
+            return PlainTextResponse("RESULTS=000")
 
-        if req.gameCd not in self.mucha_registry:
-            self.logger.warning(f"Unknown gameCd {req.gameCd}")
-            return b"RESULTS=000"
+        minfo = self.mucha_registry.get(req.gameCd, {})
+
+        if not minfo:
+            self.logger.warning(f"Unknown gameCd {req.gameCd} from {client_ip}")
+            return PlainTextResponse("RESULTS=000")
 
-        # TODO: Decrypt S/N
         b_key = b""
         for x in range(8):
             b_key += req.sendDate[(x - 1) & 7].encode()
+        
+        b_iv = b_key # what the fuck namco
 
-        cipher = Blowfish.new(b_key, Blowfish.MODE_ECB)
-        sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))
-        self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}")
+        cipher = Blowfish.new(b_key, Blowfish.MODE_CBC, b_iv)
+        try:
+            sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))[:12].decode()
+        except Exception as e:
+            self.logger.error(f"Decrypt SN {req.serialNum} failed! - {e}")
+            return PlainTextResponse("RESULTS=000")
+
+        self.logger.info(f"Boardauth request from {sn_decrypt} ({client_ip}) for {req.gameVer}")
 
         resp = MuchaAuthResponse(
-            f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}"
+            f"{self.config.server.hostname}{':' + str(self.config.server.port) if not self.config.server.is_using_proxy else ''}"
         )
 
+        netid = minfo.get('netid_prefix', "ABxN") + sn_decrypt[5:]
+
+        cab = await self.data.arcade.get_machine(netid)
+        if cab:
+            arcade = await self.data.arcade.get_arcade(cab['id'])
+            if not arcade:
+                self.logger.error(f"Failed to get arcade with id {cab['id']}")
+                return PlainTextResponse("RESULTS=000")
+
+            resp.AREA_0 = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            resp.AREA_0_EN = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            resp.AREA_FULL_0 = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            resp.AREA_FULL_0_EN = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            
+            resp.AREA_1 = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.AREA_1_EN = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.AREA_FULL_1 = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.AREA_FULL_1_EN = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+
+            resp.AREA_2 = arcade["city"] if arcade["city"] else ""
+            resp.AREA_2_EN = arcade["city"] if arcade["city"] else ""
+            resp.AREA_FULL_2 = arcade["city"] if arcade["city"] else ""
+            resp.AREA_FULL_2_EN = arcade["city"] if arcade["city"] else ""
+
+            resp.AREA_3 = ""
+            resp.AREA_3_EN = ""
+            resp.AREA_FULL_3 = ""
+            resp.AREA_FULL_3_EN = ""
+
+            resp.PREFECTURE_ID = arcade['region_id']
+            resp.COUNTRY_CD = arcade['country'] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.PLACE_ID = req.placeId if req.placeId else f"{arcade['country'] or cab['country'] or AllnetCountryCode.JAPAN.value}{arcade['id']:04X}"
+            resp.SHOP_NAME = arcade['name']
+            resp.SHOP_NAME_EN = arcade['name']
+            resp.SHOP_NICKNAME = arcade['nickname']
+            resp.SHOP_NICKNAME_EN = arcade['nickname']
+
+        elif self.config.server.allow_unregistered_serials:
+            self.logger.info(f"Allow unknown serial {netid} ({sn_decrypt}) to auth")
+        
+        else:
+            self.logger.warn(f'Auth failed for NetID {netid}')
+            return PlainTextResponse("RESULTS=000")
+
         self.logger.debug(f"Mucha response {vars(resp)}")
 
-        return self.mucha_postprocess(vars(resp))
+        return PlainTextResponse(self.mucha_postprocess(vars(resp)))
 
-    def handle_updatecheck(self, request: Request, _: Dict) -> bytes:
-        req_dict = self.mucha_preprocess(request.content.getvalue())
+    async def handle_updatecheck(self, request: Request) -> bytes:
+        bod = await request.body()
+        req_dict = self.mucha_preprocess(bod)
         client_ip = Utils.get_ip_addr(request)
 
         if req_dict is None:
             self.logger.error(
-                f"Error processing mucha request {request.content.getvalue()}"
+                f"Error processing mucha request {bod}"
             )
-            return b"RESULTS=000"
+            return PlainTextResponse("RESULTS=000")
 
         req = MuchaUpdateRequest(req_dict)
-        self.logger.info(f"Updatecheck request from {client_ip} for {req.gameVer}")        
+        self.logger.info(f"Updatecheck request from {req.serialNum} ({client_ip}) for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
 
         if req.gameCd not in self.mucha_registry:
             self.logger.warning(f"Unknown gameCd {req.gameCd}")
-            return b"RESULTS=000"
+            return PlainTextResponse("RESULTS=000")
 
-        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}")
+        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.server.hostname}{':' + str(self.config.server.port) if not self.config.server.is_using_proxy else ''}")
 
         self.logger.debug(f"Mucha response {vars(resp)}")
 
-        return self.mucha_postprocess(vars(resp))
+        return PlainTextResponse(self.mucha_postprocess(vars(resp)))
 
-    def handle_dlstate(self, request: Request, _: Dict) -> bytes:
-        req_dict = self.mucha_preprocess(request.content.getvalue())
+    async def handle_dlstate(self, request: Request) -> bytes:
+        bod = await request.body()
+        req_dict = self.mucha_preprocess(bod)
         client_ip = Utils.get_ip_addr(request)
 
         if req_dict is None:
             self.logger.error(
-                f"Error processing mucha request {request.content.getvalue()}"
+                f"Error processing mucha request {bod}"
             )
-            return b""
+            return PlainTextResponse("RESULTS=000")
         
         req = MuchaDownloadStateRequest(req_dict)
-        self.logger.info(f"DownloadState request from {client_ip} for {req.gameCd} -> {req.updateVer}")        
+        self.logger.info(f"DownloadState request from {req.serialNum} ({client_ip}) for {req.gameCd} -> {req.updateVer}")        
         self.logger.debug(f"request {vars(req)}")
-        return b"RESULTS=001"
+        return PlainTextResponse("RESULTS=001")
 
     def mucha_preprocess(self, data: bytes) -> Optional[Dict]:
         try:
@@ -169,7 +229,7 @@ class MuchaAuthResponse:
         self.RESULTS = "001"
         self.AUTH_INTERVAL = "86400"
         self.SERVER_TIME = datetime.strftime(datetime.now(), "%Y%m%d%H%M")
-        self.UTC_SERVER_TIME = datetime.strftime(datetime.now(pytz.UTC), "%Y%m%d%H%M")
+        self.SERVER_TIME_UTC = datetime.strftime(datetime.now(pytz.UTC), "%Y%m%d%H%M")
 
         self.CHARGE_URL = f"https://{mucha_url}/charge/"
         self.FILE_URL = f"https://{mucha_url}/file/"

core/templates/arcade/index.jinja

@@ -0,0 +1,19 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+{% if arcade is defined %}
+<h1>{{ arcade.name }}</h1>
+<h2>PCBs assigned to this arcade <button class="btn btn-success" id="btn_add_cab" onclick="toggle_add_cab_form()">Add</button></h2>
+{% if success is defined and success == 3 %}
+<div style="background-color: #00AA00; padding: 20px; margin-bottom: 10px; width: 15%;">
+Cab added successfully
+</div>
+{% endif %}
+<ul style="font-size: 20px;">
+{% for c in arcade.cabs %}
+<li><a href="/cab/{{ c.id }}">{{ c.serial }}</a> ({{ c.game if c.game else "Any" }})&nbsp;<button class="btn btn-secondary" onclick="prep_edit_form()">Edit</button>&nbsp;<button class="btn-danger btn">Delete</button></li>
+{% endfor %}
+</ul>
+{% else %}
+<h3>Arcade Not Found</h3>
+{% endif %}
+{% endblock content %}

core/templates/gate/create.jinja

@@ -1,4 +1,4 @@
-{% extends "core/frontend/index.jinja" %}
+{% extends "core/templates/index.jinja" %}
 {% block content %}
 <h1>Create User</h1>
 <form id="create" style="max-width: 240px; min-width: 10%;" action="/gate/gate.create" method="post">

core/templates/gate/gate.jinja

@@ -1,7 +1,7 @@
-{% extends "core/frontend/index.jinja" %}
+{% extends "core/templates/index.jinja" %}
 {% block content %}
 <h1>Gate</h1>
-{% include "core/frontend/widgets/err_banner.jinja" %}
+{% include "core/templates/widgets/err_banner.jinja" %}
 <style>
     /* Chrome, Safari, Edge, Opera */
     input::-webkit-outer-spin-button,

core/templates/index.jinja

@@ -84,7 +84,7 @@
         </style>
     </head>
     <body>
-    {% include "core/frontend/widgets/topbar.jinja" %}
+    {% include "core/templates/widgets/topbar.jinja" %}
     {% block content %}
     <h1>{{ server_name }}</h1>
     {% endblock content %}

core/templates/machine/index.jinja

@@ -0,0 +1,4 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<h1>Machine Management</h1>
+{% endblock content %}

core/templates/sys/index.jinja

@@ -0,0 +1,189 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<h1>System Management</h1>
+{% if error is defined %}
+{% include "core/templates/widgets/err_banner.jinja" %}
+{% endif %}
+<h2>Search</h2>
+<div class="row" id="rowForm">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
+            <h3>User Search</h3>
+            <div class="form-group">
+                <label for="usrId">User ID</label>
+                <input type="number" class="form-control" id="usrId" name="usrId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrName">Username</label>
+                <input type="text" class="form-control" id="usrName" name="usrName">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrEmail">Email address</label>
+                <input type="email" class="form-control" id="usrEmail" name="usrEmail">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrAc">Access Code</label>
+                <input type="text" class="form-control" id="usrAc" name="usrAc" maxlength="20" placeholder="00000000000000000000">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="shopLookup" name="shopLookup" action="/sys/lookup.shop" class="form-inline">
+            <h3>Shop search</h3>
+            <div class="form-group">
+                <label for="shopId">Shop ID</label>
+                <input type="number" class="form-control" id="shopId" name="shopId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="serialNum">Serial Number</label>
+                <input type="text" class="form-control" id="serialNum" name="serialNum" maxlength="15">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowResult" style="margin: 10px;">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for usr in usrlist %}
+        <a href=/user/{{ usr.id }}><pre>{{ usr.username if usr.username is not none else "<i>No Name Set</i>"}}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div id="shopSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for shop in shoplist %}
+        <a href="/shop/{{ shop.id }}"><pre>{{ shop.name if shop.name else "<i>No Name Set</i>"}}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+</div>
+<h2>Add</h2>
+<div class="row" id="rowAdd">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="usrAdd" name="usrAdd" action="/sys/add.user" class="form-inline" method="POST">
+            <h3>Add User</h3>
+            <div class="form-group">
+                <label for="usrName">Username</label>
+                <input type="text" class="form-control" id="usrName" name="usrName">
+            </div>
+            <br>
+            <div class="form-group">
+                <label for="usrEmail">Email address</label>
+                <input type="email" class="form-control" id="usrEmail" name="usrEmail" required>
+            </div>
+            <br>
+            <div class="form-group">
+                <label for="usrPerm">Permission Level</label>
+                <input type="number" class="form-control" id="usrPerm" name="usrPerm" value="1">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Add</button>
+        </form>
+    </div>
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="cardAdd" name="cardAdd" action="/sys/add.card" class="form-inline" method="POST">
+            <h3>Add Card</h3>
+            <div class="form-group">
+                <label for="cardUsr">User ID</label>
+                <input type="number" class="form-control" id="cardUsr" name="cardUsr" required>
+            </div>
+            <br>
+            <div class="form-group">
+                <label for="cardAc">Access Code</label>
+                <input type="text" class="form-control" id="cardAc" name="cardAc" maxlength="20" placeholder="00000000000000000000" required>
+            </div>
+            <br>
+            <div class="form-group">
+                <label for="cardIdm">IDm/Chip ID</label>
+                <input type="text" class="form-control" id="cardIdm" name="cardIdm" disabled>
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Add</button>
+        </form>
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="shopAdd" name="shopAdd" action="/sys/add.shop" class="form-inline" method="POST">
+            <h3>Add Shop</h3>
+            <div class="form-group">
+                <label for="shopName">Name</label>
+                <input type="text" class="form-control" id="shopName" name="shopName">
+            </div>
+            <br>
+            <div class="form-group">
+                <label for="shopCountry">Country Code</label>
+                <input type="text" class="form-control" id="shopCountry" name="shopCountry" maxlength="3" placeholder="JPN">
+            </div>
+            <br />
+            <div class="form-group">
+                <label for="shopIp">VPN IP</label>
+                <input type="text" class="form-control" id="shopIp" name="shopIp">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Add</button>
+        </form>
+    </div>
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="cabAdd" name="cabAdd" action="/sys/add.cab" class="form-inline" method="POST">
+            <h3>Add Machine</h3>
+            <div class="form-group">
+                <label for="cabShop">Shop ID</label>
+                <input type="number" class="form-control" id="cabShop" name="cabShop" required>
+            </div>
+            <br>
+            <div class="form-group">
+                <label for="cabSerial">Serial</label>
+                <input type="text" class="form-control" id="cabSerial" name="cabSerial">
+            </div>
+            <br />
+            <div class="form-group">
+                <label for="cabGame">Game Code</label>
+                <input type="text" class="form-control" id="cabGame" name="cabGame" maxlength="4" placeholder="SXXX">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Add</button>
+        </form>
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowAddResult" style="margin: 10px;">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div id="userAddResult" class="col-sm-6" style="max-width: 25%;">
+        {% if usradd is defined %}
+            <pre>Added user {{ usradd.username if usradd.username is not none else "with no name"}} with id {{usradd.id}} and password {{ usradd.password }}</pre>
+        {% endif %}
+    </div>
+    <div id="cardAddResult" class="col-sm-6" style="max-width: 25%;">
+        {% if cardadd is defined %}
+            <pre>Added {{ cardadd.access_code }} with id {{cardadd.id}} to user {{ cardadd.user }}</pre>
+        {% endif %}
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div id="shopAddResult" class="col-sm-6" style="max-width: 25%;">
+        {% if shopadd is defined %}
+            <pre>Added Shop {{ shopadd.id }}</pre></a>
+        {% endif %}
+    </div>
+    <div id="cabAddResult" class="col-sm-6" style="max-width: 25%;">
+        {% if cabadd is defined %}
+            <pre>Added Machine {{ cabadd.id }} with serial {{ cabadd.serial }}</pre></a>
+        {% endif %}
+    </div>
+    {% endif %}
+</div>
+{% endblock content %}

core/templates/user/index.jinja

@@ -0,0 +1,175 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<script type="text/javascript">
+function toggle_new_name_form() {
+  let frm = document.getElementById("new_name_form");
+  let btn = document.getElementById("btn_toggle_form");
+
+  if (frm.style['display'] != "") {
+    frm.style['display'] = "";
+    frm.style['max-height'] = "";
+    btn.innerText = "Cancel";
+  } else {
+    frm.style['display'] = "none";
+    frm.style['max-height'] = "0px";
+    btn.innerText = "Edit";
+  }
+}
+function toggle_add_card_form() {
+  let btn = document.getElementById("btn_add_card");
+  let dv = document.getElementById("add_card_container")
+
+  if (dv.style['display'] != "") {
+    btn.innerText = "Cancel";
+    dv.style['display'] = "";
+  } else {
+    btn.innerText = "Add";
+    dv.style['display'] = "none";
+  }
+}
+
+function prep_edit_form(access_code, chip_id, idm, card_type, u_memo) {
+  ac = document.getElementById("card_edit_frm_access_code");
+  cid = document.getElementById("card_edit_frm_chip_id");
+  fidm = document.getElementById("card_edit_frm_idm");
+  memo = document.getElementById("card_edit_frm_memo");
+
+  if (chip_id == "None" || chip_id == undefined) {
+    chip_id = ""
+  }
+  if (idm == "None" || idm == undefined) {
+    idm = ""
+  }
+  if (u_memo == "None" || u_memo == undefined) {
+    u_memo = ""
+  }
+
+  ac.value = access_code;
+  cid.value = chip_id;
+  fidm.value = idm;
+  memo.value = u_memo;
+
+  if (card_type == "AmusementIC") {
+    cid.disabled = true;
+    fidm.disabled = false;
+  } else {
+    cid.disabled = false;
+    fidm.disabled = true;
+  }
+}
+</script>
+<h1>Management for {{ username }}&nbsp;<button onclick="toggle_new_name_form()" class="btn btn-secondary" id="btn_toggle_form">Edit</button></h1>
+{% if error is defined %}
+{% include "core/templates/widgets/err_banner.jinja" %}
+{% endif %}
+{% if success is defined and success == 2 %}
+<div style="background-color: #00AA00; padding: 20px; margin-bottom: 10px; width: 15%;">
+Update successful
+</div>
+{% endif %}
+<form style="max-width: 33%; display: none; max-height: 0px;" action="/user/update.name" method="post" id="new_name_form">
+  <div class="mb-3">
+    <label for="new_name" class="form-label">New Nickname</label>
+    <input type="text" class="form-control" id="new_name" name="new_name" aria-describedby="new_name_help">
+    <div id="new_name_help" class="form-text">Must be 10 characters or less</div>
+  </div>
+  <button type="submit" class="btn btn-primary">Submit</button>
+</form>
+<p></p>
+<h2>Cards <button class="btn btn-success" id="btn_add_card" onclick="toggle_add_card_form()">Add</button></h2>
+{% if success is defined and success == 3 %}
+<div style="background-color: #00AA00; padding: 20px; margin-bottom: 10px; width: 15%;">
+Card added successfully
+</div>
+{% endif %}
+<div id="add_card_container" style="display: none; max-width: 33%;">
+  <form action="/user/add.card" method="post", id="frm_add_card">
+    <label class="form-label" for="card_add_frm_access_code">Access Code:</label>
+    <input class="form-control" name="add_access_code" id="card_add_frm_access_code" maxlength="20" type="text" required aria-describedby="ac_help">
+    <div id="ac_help" class="form-text">20 digit code on the back of the card.</div>
+    <button type="submit" class="btn btn-primary">Add</button>
+  </form>
+  <br>
+</div>
+<ul style="font-size: 20px;">
+{% for c in cards %}
+<li>{{ c.access_code }} ({{ c.type}}): {{ c.status }}&nbsp;<button onclick="prep_edit_form('{{ c.access_code }}', '{{ c.chip_id}}', '{{ c.idm }}', '{{ c.type }}', '{{ c.memo }}')" data-bs-toggle="modal" data-bs-target="#card_edit" class="btn btn-secondary" id="btn_edit_card_{{ c.access_code }}">Edit</button>&nbsp;{% if c.status == 'Active'%}<button class="btn-warning btn">Lock</button>{% elif c.status == 'Locked' %}<button class="btn-warning btn">Unlock</button>{% endif %}&nbsp;<button class="btn-danger btn">Delete</button></li>
+{% endfor %}
+</ul>
+
+<h2>Reset Password</h2>
+{% if success is defined and success == 1 %}
+<div style="background-color: #00AA00; padding: 20px; margin-bottom: 10px; width: 15%;">
+Update successful
+</div>
+{% endif %}
+<form style="max-width: 33%;" action="/user/update.pw" method="post">
+  <div class="mb-3">
+    <label for="current_pw" class="form-label">Current Password</label>
+    <input type="password" class="form-control" id="current_pw" name="current_pw">
+  </div>
+  <div class="mb-3">
+    <label for="password1" class="form-label">New Password</label>
+    <input type="password" class="form-control" id="password1" name="password1" aria-describedby="password_help">
+    <div id="password_help" class="form-text">Password must be at least 10 characters long, contain an upper and lowercase character, number, and special character</div>
+  </div>
+  <div class="mb-3">
+    <label for="password2" class="form-label">Retype New Password</label>
+    <input type="password" class="form-control" id="password2" name="password2">
+  </div>
+  <button type="submit" class="btn btn-primary">Submit</button>
+</form>
+
+{% if arcades is defined and arcades|length > 0 %}
+<h2>Arcades</h2>
+  <ul>
+  {% for a in arcades %}
+    <li><h3>{{ a.name }}</h3>
+      {% if a.machines|length > 0 %}
+        <table>
+          <tr><th>Serial</th><th>Game</th><th>Last Seen</th></tr>
+          {% for m in a.machines %}
+            <tr><td>{{ m.serial }}</td><td>{{ m.game }}</td><td>{{ m.last_seen }}</td></tr>
+          {% endfor %}
+        </table>
+      {% endif %}
+    </li>
+  {% endfor %}
+  </ul>
+{% endif %}
+
+<div class="modal fade" id="card_edit" tabindex="-1" aria-labelledby="card_edit_label" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h1 class="modal-title fs-5" id="card_edit_label">Edit Card</h1>
+        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+      </div>
+      <div class="modal-body">
+        <form action="/user/edit.card" method="post" id="frm_edit_card">
+          <label class="form-label" for="card_edit_frm_access_code">Access Code:</label>
+          <input class="form-control" readonly name="add_access_code" id="card_edit_frm_access_code" maxlength="20" type="text" required aria-describedby="ac_help">
+          <div id="ac_help" class="form-text">20 digit code on the back of the card. If this is incorrect, contact a sysadmin.</div>
+
+          <label class="form-label" for="card_edit_frm_memo" id="card_edit_frm_memo_lbl">Memo:</label>
+          <input class="form-control" aria-describedby="memo_help" name="add_memo" id="card_edit_frm_memo" maxlength="16" type="text">
+          <div id="memo_help" class="form-text">Must be 16 characters or less.</div>
+
+          <label class="form-label" for="card_edit_frm_idm" id="card_edit_frm_idm_lbl">FeliCa IDm:</label>
+          <input class="form-control" aria-describedby="idm_help" name="add_felica_idm" id="card_edit_frm_idm" maxlength="16" type="text">
+          <div id="idm_help" class="form-text">8 bytes that uniquly idenfites a FeliCa card. Obtained by reading the card with an NFC reader.</div>
+        
+          <label class="form-label" for="card_edit_frm_chip_id" id="card_edit_frm_chip_id_lbl">Mifare UID:</label>
+          <input class="form-control" aria-describedby="chip_id_help" name="add_mifare_chip_id" id="card_edit_frm_chip_id" maxlength="8" type="text">
+          <div id="chip_id_help" class="form-text">4 byte integer that uniquly identifies a Mifare card. Obtained by reading the card with an NFC reader.</div>
+        </form>
+      </div>
+      <div class="modal-footer">
+        <button type="submit" class="btn btn-primary" form="frm_edit_card">Edit</button>
+        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+      </div>
+    </div>
+  </div>
+</div>
+
+{% endblock content %}

core/templates/widgets/err_banner.jinja

@@ -0,0 +1,28 @@
+{% if error > 0 %}
+<div class="err-banner">
+<h3>Error</h3>
+{% if error == 1 %}
+Card not registered, or wrong password
+{% elif error == 2 %}
+Missing or malformed access code
+{% elif error == 3 %}
+Failed to create user
+{% elif error == 4 %}
+Required field not filled or invalid
+{% elif error == 5 %}
+Incorrect old password
+{% elif error == 6 %}
+Passwords don't match
+{% elif error == 7 %}
+New password not acceptable
+{% elif error == 8 %}
+New Nickname too long
+{% elif error == 9 %}
+You must be logged in to preform this action
+{% elif error == 10 %}
+Invalid serial number
+{% else %}
+An unknown error occoured
+{% endif %}
+</div>
+{% endif %}

core/templates/widgets/topbar.jinja

@@ -3,19 +3,20 @@
 </div>
 <div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
     <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
-    {% for game in game_list %}    
-    <a href=/game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
+    {% for game, data in game_list|items %}    
+    <a href=/game{{ data.url }}/><button class="btn btn-success">{{ game }}</button></a>&nbsp;
     {% endfor %}
 </div>
 </div>
 <div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
     {% if sesh is defined and sesh["permissions"] >= 2 %}
-    <a href="/sys"><button class="btn btn-primary">System</button></a>
+    <a href="/sys/"><button class="btn btn-primary">System</button></a>
     {% endif %}
-    {% if sesh is defined and sesh["userId"] > 0 %}
-    <a href="/user"><button class="btn btn-primary">Account</button></a>
+    {% if sesh is defined and sesh["user_id"] > 0 %}
+    <a href="/user/"><button class="btn btn-primary">Account</button></a>
+    <a href="/user/logout"><button class="btn btn-danger">Logout</button></a>
     {% else %}
-    <a href="/gate"><button class="btn btn-primary">Gate</button></a>
+    <a href="/gate/"><button class="btn btn-primary">Gate</button></a>
     {% endif %}
     
 </div>

core/title.py

@@ -1,12 +1,24 @@
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Any
+import json
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web.http import Request
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Route
 
 from core.config import CoreConfig
 from core.data import Data
 from core.utils import Utils
 
+class JSONResponseNoASCII(Response):
+    media_type = "application/json"
+
+    def render(self, content: Any) -> bytes:
+        return json.dumps(
+            content,
+            ensure_ascii=False,
+        ).encode("utf-8")
+
 class BaseServlet:
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
         self.core_cfg = core_cfg
@@ -28,18 +40,16 @@ class BaseServlet:
         """
         return False
     
-    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
+    def get_routes(self) -> List[Route]:
         """Called during boot to get all matcher endpoints this title servlet handles
 
         Returns:
-            Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]: A 2-length tuple where offset 0 is GET and offset 1 is POST, 
-            containing a list of 3-length tuples where offset 0 is the name of the function in the handler that should be called, offset 1
-            is the matching string, and offset 2 is a dict containing rules for the matcher.
+            List[Route]: A list of Routes, WebSocketRoutes, or similar classes
         """
-        return (
-            [("render_GET", "/{game}/{version}/{endpoint}", {'game': R'S...'})], 
-            [("render_POST", "/{game}/{version}/{endpoint}", {'game': R'S...'})]
-        )
+        return [
+            Route("/{game}/{version}/{endpoint}", self.render_POST, methods=["POST"]),
+            Route("/{game}/{version}/{endpoint}", self.render_GET, methods=["GET"]),
+        ]
     
     def setup(self) -> None:
         """Called once during boot, should contain any additional setup the handler must do, such as starting any sub-services
@@ -58,11 +68,11 @@ class BaseServlet:
             Tuple[str, str]: A tuple where offset 0 is the allnet uri field, and offset 1 is the allnet host field
         """
         if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
-            return (f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", "")
+            return (f"http://{self.core_cfg.server.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", "")
 
-        return (f"http://{self.core_cfg.title.hostname}/{game_code}/{game_ver}/", "")
+        return (f"http://{self.core_cfg.server.hostname}/{game_code}/{game_ver}/", "")
 
-    def get_mucha_info(self, core_cfg: CoreConfig, cfg_dir: str) -> Tuple[bool, str]:
+    def get_mucha_info(self, core_cfg: CoreConfig, cfg_dir: str) -> Tuple[bool, List[str], List[str]]:
         """Called once during boot to check if this game is a mucha game
 
         Args:
@@ -72,15 +82,15 @@ class BaseServlet:
         Returns:
             Tuple[bool, str]: Tuple where offset 0 is true if the game is enabled, false otherwise, and offset 1 is the game CD
         """
-        return (False, "")
+        return (False, [], [])
 
-    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
-        self.logger.warn(f"{game_code} Does not dispatch POST")
-        return None
+    async def render_POST(self, request: Request) -> bytes:
+        self.logger.warn(f"Game Does not dispatch POST")
+        return Response()
 
-    def render_GET(self, request: Request, game_code: str, matchers: Dict) -> bytes:
-        self.logger.warn(f"{game_code} Does not dispatch GET")
-        return None
+    async def render_GET(self, request: Request) -> bytes:
+        self.logger.warn(f"Game Does not dispatch GET")
+        return Response()
 
 class TitleServlet:
     title_registry: Dict[str, BaseServlet] = {}
@@ -136,7 +146,7 @@ class TitleServlet:
                 self.logger.error(f"{folder} missing game_code or index in __init__.py, or is_game_enabled in index")
 
         self.logger.info(
-            f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.title.port) if core_cfg.title.port > 0 else ''}"
+            f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.server.port) if core_cfg.server.port > 0 else ''}"
         )
 
     def render_GET(self, request: Request, endpoints: dict) -> bytes:

core/utils.py

@@ -1,6 +1,6 @@
 from typing import Dict, Any, Optional
 from types import ModuleType
-from twisted.web.http import Request
+from starlette.requests import Request
 import logging
 import importlib
 from os import walk
@@ -34,33 +34,22 @@ class Utils:
 
     @classmethod
     def get_ip_addr(cls, req: Request) -> str:
-        return (
-            req.getAllHeaders()[b"x-forwarded-for"].decode()
-            if b"x-forwarded-for" in req.getAllHeaders()
-            else req.getClientAddress().host
-        )
+        ip = req.headers.get("x-forwarded-for", req.client.host)
+        return ip.split(", ")[0]
     
     @classmethod
     def get_title_port(cls, cfg: CoreConfig):
         if cls.real_title_port is not None: return cls.real_title_port
 
-        if cfg.title.port == 0:
-            cls.real_title_port = cfg.allnet.port
-        
-        else:
-            cls.real_title_port = cfg.title.port
+        cls.real_title_port = cfg.server.proxy_port if cfg.server.is_using_proxy and cfg.server.proxy_port else cfg.server.port
         
         return cls.real_title_port
-
+    
     @classmethod
     def get_title_port_ssl(cls, cfg: CoreConfig):
         if cls.real_title_port_ssl is not None: return cls.real_title_port_ssl
 
-        if cfg.title.port_ssl == 0:
-            cls.real_title_port_ssl = 443
-        
-        else:
-            cls.real_title_port_ssl = cfg.title.port_ssl
+        cls.real_title_port_ssl = cfg.server.proxy_port_ssl if cfg.server.is_using_proxy and cfg.server.proxy_port_ssl else Utils.get_title_port(cfg)
         
         return cls.real_title_port_ssl
 

dbutils.py

@@ -1,9 +1,12 @@
-import yaml
+#!/usr/bin/env python3
 import argparse
 import logging
-from core.config import CoreConfig
+from os import mkdir, path, access, W_OK
+import yaml
+import asyncio
+
 from core.data import Data
-from os import path, mkdir, access, W_OK
+from core.config import CoreConfig
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Database utilities")
@@ -16,19 +19,10 @@ if __name__ == "__main__":
         type=str,
         help="Version of the database to upgrade/rollback to",
     )
-    parser.add_argument(
-        "--game",
-        "-g",
-        type=str,
-        help="Game code of the game who's schema will be updated/rolled back. Ex. SDFE",
-    )
     parser.add_argument("--email", "-e", type=str, help="Email for the new user")
-    parser.add_argument("--old_ac", "-o", type=str, help="Access code to transfer from")
-    parser.add_argument("--new_ac", "-n", type=str, help="Access code to transfer to")
-    parser.add_argument("--force", "-f", type=bool, help="Force the action to happen")
-    parser.add_argument(
-        "action", type=str, help="DB Action, create, recreate, upgrade, or rollback"
-    )
+    parser.add_argument("--access_code", "-a", type=str, help="Access code for new/transfer user", default="00000000000000000000")
+    parser.add_argument("--message", "-m", type=str, help="Revision message")
+    parser.add_argument("action", type=str, help="create, upgrade, downgrade, create-owner, migrate, create-revision, create-autorevision")
     args = parser.parse_args()
 
     cfg = CoreConfig()
@@ -50,42 +44,31 @@ if __name__ == "__main__":
 
     if args.action == "create":
         data.create_database()
+    
+    elif args.action == "upgrade":
+        data.schema_upgrade(args.version)
 
-    elif args.action == "recreate":
-        data.recreate_database()
-
-    elif args.action == "upgrade" or args.action == "rollback":
-        if args.version is None:
-            data.logger.warning("No version set, upgrading to latest")
-
-        if args.game is None:
-            data.logger.warning("No game set, upgrading core schema")
-            data.migrate_database(
-                "CORE",
-                int(args.version) if args.version is not None else None,
-                args.action,
-            )
-
-        else:
-            data.migrate_database(
-                args.game,
-                int(args.version) if args.version is not None else None,
-                args.action,
-            )
-
-    elif args.action == "autoupgrade":
-        data.autoupgrade()
+    elif args.action == "downgrade":
+        if not args.version:
+            logging.getLogger("database").error(f"Version argument required for downgrade")
+            exit(1)
+        data.schema_downgrade(args.version)
 
     elif args.action == "create-owner":
-        data.create_owner(args.email)
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.create_owner(args.email, args.access_code))
 
-    elif args.action == "migrate-card":
-        data.migrate_card(args.old_ac, args.new_ac, args.force)
+    elif args.action == "migrate":
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.migrate())
 
-    elif args.action == "cleanup":
-        data.delete_hanging_users()
-    
-    elif args.action == "version":
-        data.show_versions()
+    elif args.action == "create-revision":
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.create_revision(args.message))
 
-    data.logger.info("Done")
+    elif args.action == "create-autorevision":
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.create_revision_auto(args.message))
+
+    else:
+        logging.getLogger("database").info(f"Unknown action {args.action}")

docs/INSTALL_LINUX.md

@@ -0,0 +1,107 @@
+# Installing ARTEMiS on Linux
+This guide assumes a fresh install of Debian 12 or Rasperry Pi OS. If you're using a different distrubution, your package manager commands and package names may be different then what's listed below. Please check with your repository's package manager for package names.
+
+## Install prerequisits
+### Python
+Some installs may come with python already installed. You can verify this by trying the following commands:
+- `python --version`
+- `python3 --version`
+- `python3.<minor version> --version` where `<minor version>` is a python 3 release (eg 11, 10)
+
+If your python version is at least 3.7, you can move to the next step
+
+### Libraries and other software
+ARTEMiS depends on mysql and memcached. As stated above, package names may vary by distrubution, but this is generally what you should expect to install.
+#### Rasperry Pi OS
+`sudo apt install git mariadb-server python3-pip memcached libmemcached-dev `
+
+#### Debian 12
+`sudo apt install git mariadb-server python3-pip memcached libmemcached-dev default-libmysqlclient-dev pkg-config`
+
+### Optional: Install proxy
+If you intend to use a proxy (recomended for public-facing production setups), we recomend nginx
+`sudo apt install nginx`
+
+## Database setup
+### mysql_secure_installation
+If you already have your database installed and configured, and are able to log in, skip down to the [Creating the database](#creating-the-database) section below. Otherwise, setup your newly installed database.
+
+`sudo mysql_secure_installation`
+
+Leave the root password blank, do not switch to unix socket, do reset the root password to something secure, and answer yes to the rest of the prompts. You can then log into your database with `sudo mysql`
+
+### Creating the database
+Once you're logged in, run the following commands, as root, to set up our database. Make sure you note down whatever you decide to make the password for the aime account, as you will need it to configure artemis.
+
+```sql
+CREATE USER 'aime'@'localhost' IDENTIFIED BY '<password>';
+CREATE DATABASE aime;
+GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
+quit
+```
+We have now set up our new user, `aime`, created a database called `aime` and given our user all the permissions it needs on every table of that database.
+
+### Configure memcached
+Under the file /etc/memcached.conf, please make sure the following parameters are set:
+
+```
+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
+# Note that the daemon will grow to this size, but does not start out holding this much
+# memory
+
+-I 128m
+-m 1024
+```
+
+** This is mandatory to avoid memcached overload caused by Crossbeats or by massive profiles
+
+Restart memcached using:  sudo systemctl restart memcached
+
+## Getting ARTEMiS
+### Clone from gitea
+use `git clone https://gitea.tendokyu.moe/Hay1tsme/artemis.git` to pull down ARTEMiS into a folder called `artemis` created at wherever your current working directory is. `cd` into `artemis`.
+
+### Optional: Create a venv
+Python venvs are a way to install and manage packages on a per-project basis and are recomended on systems that will have multiple python scripts running on them to avoid dependancy issues. If this server will be running ARTEMiS and ONLY ARTEMiS, then it is possible to get away without creating one. If you do want to create one, you will have to install an additional package:
+
+`sudo apt install python3-venv` (like above, package name may vary depending on distro and python version)
+
+Now, simply run `python -m venv .venv` (may have to use python3 or python 3.11 instead of python) to create your virtual environment in the folder `.venv`. In order to install packages and run scripts in this environment, you have to 'activate' it by running `source .venv/bin/activate`. Your terminal should now have (venv) appended to it.
+
+### Optional: Use the develop branch
+By default, pulling down ARTEMiS from gitea will pull the `master` branch. This branch is updated less frequently, but is considered stable and ready for production use. If you'd rather have more updates, but a possibility for instability or bugs, you can switch to the develop branch by running `git checkout develop`. You can run `git checkout master` to switch back to stable.
+
+## Install python libraries
+Run `pip install -r requirements.txt` to install all of ARTEMiS' dependencies. If any installs fail, you may have missed a step in the [Install prerequisits](#install-prerequisits) section above. If you're absolutly sure you didn't, submit an issue on gitea.
+
+## Configuration
+### Copy example configs
+From the `artemis` directory, run `cp -r example_config config` to copy the example configuration files to a new folder called `config`. All of the config changes you make will be done in the `config` folder.
+
+### Optional: Generate AimeDB and Frontend JWT Secrets
+AimeDB and the frontend utalize JSON Web Tokens (JWT) for card authentication and session cookies respectivly. While generating a secret for AimeDB is optional, if you intend to run the frontend, a secret is required. You can generate a secret easily by running:
+
+`openssl rand --base64 64`
+
+With 64 being the number of bytes. You shouldn't need to go higher then 64, but you can if desired. **NOTE: When pasting secrets into the config file, make sure you remove any newlines!**
+
+### Edit `core.yaml`
+Before editing `core.yaml`, you should familiarize yourself with the name and function of each of the config options. You can find a full list in [config.md](config.md)
+
+Open `core.yaml` in the `config` folder in your prefered text editor. The only configuration option that it is absolutly mandatory to change is `aimedb`->`key`. This key must be set for the server to start, and the key must be correct, otherwise you will not be able to process aimedb requests. The correct key is floating around online, and finding it is left as an excersie to the reader. 
+
+Another option that should be changed is `database`->`password` to be the password you set when you created your database user. You did write it down somewhere, right?
+
+Since you are presumably not running the games on the same computer you're installing this server on, you're going to want to change `server`->`hostname` to be whatever hostname or IP address other PCs can reach this server by. Note that some games reject IPs and require hostnames, so setting a hostname is always recomended over an IP.
+
+### Edit game configs
+Every game has their own yaml file with settings that you may want to tweek. `InitialD Zero` and `Pokken` both have `hostname` fields in their config file that you should edit, and some games support encryption, if supplied with proper keys.
+
+### A note about IDZ
+InitialD Zero is currently the only game where it is required to specify encryption information (the AES key and at least one RSA key) for the game to start. These keys are, like the aimedb key, floating around online and will not be provided. If you don't have the keys, and don't plan on anybody connecting to your server playing InitialD Zero, it's best to set `enabled` to `False` in idz.yaml to disable the game.
+
+## Create database tables
+ARTEMiS uses alembic to manage datbase versioning. `dbutils.py` acts as a wrapper for alembic, and can execute some necessassary database functions. To create the database tables, run `python dbutils.py create`. Confirm that there are no errors, and you're good to go. If you intend to use the frontend, you may also want to run `python dbutils.py create-owner -a <your 20 digit access code here>` to create a superuser account to log in with.
+
+## Run ARTEMiS
+Once you have everything configured properly, simply run `python index.py` to start ARTEMiS. Verify that clients can connect to all services (allnet, billing, aimedb, and game servers) and setup is complete.

docs/INSTALL_WINDOWS.md

@@ -1,102 +1,77 @@
-# ARTEMiS - Windows 10/11 Guide
-This step-by-step guide assumes that you are using a fresh install of Windows 10/11 without MySQL installed, some of the steps can be skipped if you already have an installation with MySQL 8.0 or even some of the modules already present on your environment
+# Installing ARTEMiS on Windows
+This guide assumes a fresh install of Windows 10. Please be aware that due to the lack of memcached and the general woes of running a server on Windows, this is only recommended for local setups or small hosting-for-the-homies type servers.
 
-# Setup
-## Install Python Python 3.9 (recommended) or 3.10
-1. Download Python 3.9 : [Link](https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe)
-2. Install python-3.9.13-amd64.exe
-    1. Select Customize installation
-    2. Make sure that pip, tcl/tk, and the for all users are checked and hit Next
-    3. Make sure that you enable "Create shortcuts for installed applications" and "Add Python to environment variables" and hit Install
+## Install prerequisites
+### Python
+- Python versions from 3.8 to 3.11 work with ARTEMiS. We recommend 3.11.
+    - https://www.python.org/ftp/python/3.11.7/python-3.11.7-amd64.exe
+- Install using whichever options best suit your environment, making sure that the Python executable is on path, such that you can open CMD, type `python --version` and see the version of Python you have installed.
+- If you already have a working version of Python installed, skip this step.
 
-## Install MySQL 8.0
-1. Download MySQL 8.0 Server : [Link](https://dev.mysql.com/get/Downloads/MySQLInstaller/mysql-installer-community-8.0.34.0.msi)
-2. Install mysql-installer-web-community-8.0.34.0.msi
-    1. Click on "Add ..." on the side
-    2. Click on the "+" next to MySQL Servers
-    3. Make sure MySQL Server 8.0.34 - X64 is under the products to be installed.
-    4. Hit Next and Next once installed
-    5. Select the configuration type "Development Computer"
-    6. Hit Next
-    7. Select "Use Legacy Authentication Method (Retain MySQL 5.x compatibility)" and hit Next
-    8. Enter a root password and then hit Next >
-    9. Leave everything under Windows Service as default and hit Next >
-    10. Click on Execute and for it to finish and hit Next> and then Finish
-3. Open MySQL 8.0 Command Line Client and login as your root user
-4. Change `<Enter Password Here>` to a new password for the user aime, type those commands to create your user and the database
+### MariaDB
+- It is always recommended to use MariaDB over MySQL because Oracle is a terrible company.
+- While the latest release of v10 is recommended, as it is an LTS release, v11 should work fine.
+    - https://ftp.osuosl.org/pub/mariadb//mariadb-10.11.6/winx64-packages/mariadb-10.11.6-winx64.msi
+- REMEMBER YOUR ROOT PASSWORD SO YOU CAN LOG IN IN FUTURE STEPS.
 
-```sql
-CREATE USER 'aime'@'localhost' IDENTIFIED BY '<Enter Password Here>';
+### Git
+- While technically optional, it is strongly recommended to obtain ARTEMiS via git clone instead of just downloading it.
+    - https://git-scm.com/download/win
+- It is recommended to use Notepad++ as the default editor (if you have it installed), other than that, the default settings should be fine.
+
+### Optional: GUI database viewer
+- Having a GUI database editor is recommended but not required.
+- MariaDB will try to install HeidiSQL, but we recommend DBeaver.
+    - https://dbeaver.io/download/
+
+## Obtain ARTEMiS
+### Via git (recommended)
+- `git clone https://gitea.tendokyu.moe/Hay1tsme/artemis.git` via cmd in whatever folder you want to install ARTEMiS.
+    - You can switch to the develop branch for latest changes via `git checkout develop`.
+
+### Via http download
+- Download [here](https://gitea.tendokyu.moe/Hay1tsme/artemis/archive/master.zip).
+    - Develop branch can be found [here](https://gitea.tendokyu.moe/Hay1tsme/artemis/archive/develop.zip).
+- Extract the zip file somewhere.
+
+## Database setup
+- Log into your server as root, either via GUI (recommended) or CMD
+- Create the `aime` user, replace `<password>` with a password you choose. Remember it!
+```
+CREATE USER 'aime'@'localhost' IDENTIFIED BY '<password>';
 CREATE DATABASE aime;
 GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
-FLUSH PRIVILEGES;
-exit;
 ```
+- If you create the database via a GUI, make sure you grant all the above permissions.
 
-## Install Python modules
-1. Change your work path to the artemis-master folder using 'cd' and install the requirements:
+## Create a venv
+- Python virtual environments are a good way to manage packages and make dealing with python and pip easier.
+- `python -m pip venv venv`
+- `venv\Scripts\activate.bat` to activate the venv whenever you need to interact with ARTEMiS.
+- All the rest of the steps assume your venv is activated.
 
-```shell
-pip install -r requirements.txt
-```	
+## Install pip modules
+- `pip install -r requirements.txt`
 
-## Copy/Rename the folder `example_config` to `config`
+## Setup configuration
+- Create a new `config` folder and copy the files in `example_config` over.
+- edit `core.yaml`
+    - Put the password you created for the aime user into the `database` section.
+    - Put in the aimedb key (YOU DO NOT GENERATE THIS KEY, FIND IT SOMEWHERE).
+    - Set your hostname to be whatever hostname or IP address games can reach your server at (many games reject localhost and 127.0.0.1).
+    - Optional: generate base64-encoded secrets for aimedb and frontend.
+    - See [config.md](docs/config.md) for a full list of options.
+- edit `idz.yaml`
+    - If you don't plan on anyone using your server to play Initial D Zero, it is best to disable it to cut down on console spam on boot.
+- Edit other game yamls
+    - Add keys, set hostnames, ports, etc. Specific settings will depend on the game. See [game_specific_info](docs/game_specific_info.md).
 
-## Adjust `config/core.yaml`
+## Create Database Tables
+- `python dbutils.py create`
 
-1. Make sure to change the server `hostname` to be set to your local machine IP (ex.: 192.168.xxx.xxx) 
-    - In case you want to run this only locally, set the following values:
+## Firewall
+- If you're planning on serving games not on your PC, open at least ports 80, 8443, and 22345 in windows firewall
+    - Also set `listen_address` to either your local IP to serve on your LAN, or `0.0.0.0` for all interfaces, to accept connections from other places.
 
-```yaml
-server:
-    listen_address: 0.0.0.0
-title: 
-    hostname: 192.168.xxx.xxx
-```
-
-1. Adjust the proper MySQL information you created earlier
-```yaml
-database:
-  host: "localhost"
-  username: "aime"
-  password: "<Enter Password Here>"
-  name: "aime"
-```
-3. Add the AimeDB key at the bottom of the file
-4. If the webui is needed, change the flag from False to True
-
-## Create the database tables for ARTEMiS
-
-```shell
-python dbutils.py create
-```
-
-## Firewall Adjustements 
-Make sure the following ports are open both on your router and local Windows firewall in case you want to use this for public use (NOT recommended):
-> Port 80 (TCP), 443 (TCP), 8443 (TCP), 22345 (TCP), 8080 (TCP), 8090 (TCP) **webui, 8444 (TCP) **mucha
-
-## Running the ARTEMiS instance
-```shell
-python index.py
-```
-
-# Troubleshooting
-
-## Game does not connect to ARTEMiS Allnet server
-1. Double-check your core.yaml, the listen_address is most likely either not binded to the proper IP or the port is not opened
-
-## Game does not connect to Title Server
-1. Verify that your core.yaml is setup properly for both the server listen_address and title hostname
-2. Boot your game and verify that an AllNet response does show and if it does, attempt to open the URI that is shown under a browser such as Edge, Chrome & Firefox.
-3. If a page is shown, the server is working properly and if it doesn't, double check your port forwarding and also that you have entered the proper local IP under the Title hostname in core.yaml.
-
-## Unhandled command under AimeDB
-1. Double check your AimeDB key under core.yaml, it is incorrect.
-
-## AttributeError: module 'collections' has no attribute 'Hashable'
-1. This means the pyYAML module is obsolete, simply rerun pip with the -U (force update) flag, as shown below.
-    - Change your work path to the artemis-master (or artemis-develop) folder using 'cd' and run the following commands:
-
-```shell
-pip install -r requirements.txt -U
-```
+## Start ARTEMiS
+- `python index.py`

docs/INSTALL_WINDOWS_old.md

@@ -0,0 +1,102 @@
+# ARTEMiS - Windows 10/11 Guide
+This step-by-step guide assumes that you are using a fresh install of Windows 10/11 without MySQL installed, some of the steps can be skipped if you already have an installation with MySQL 8.0 or even some of the modules already present on your environment
+
+# Setup
+## Install Python Python 3.9 (recommended) or 3.10
+1. Download Python 3.9 : [Link](https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe)
+2. Install python-3.9.13-amd64.exe
+    1. Select Customize installation
+    2. Make sure that pip, tcl/tk, and the for all users are checked and hit Next
+    3. Make sure that you enable "Create shortcuts for installed applications" and "Add Python to environment variables" and hit Install
+
+## Install MySQL 8.0
+1. Download MySQL 8.0 Server : [Link](https://dev.mysql.com/get/Downloads/MySQLInstaller/mysql-installer-community-8.0.34.0.msi)
+2. Install mysql-installer-web-community-8.0.34.0.msi
+    1. Click on "Add ..." on the side
+    2. Click on the "+" next to MySQL Servers
+    3. Make sure MySQL Server 8.0.34 - X64 is under the products to be installed.
+    4. Hit Next and Next once installed
+    5. Select the configuration type "Development Computer"
+    6. Hit Next
+    7. Select "Use Legacy Authentication Method (Retain MySQL 5.x compatibility)" and hit Next
+    8. Enter a root password and then hit Next >
+    9. Leave everything under Windows Service as default and hit Next >
+    10. Click on Execute and for it to finish and hit Next> and then Finish
+3. Open MySQL 8.0 Command Line Client and login as your root user
+4. Change `<Enter Password Here>` to a new password for the user aime, type those commands to create your user and the database
+
+```sql
+CREATE USER 'aime'@'localhost' IDENTIFIED BY '<Enter Password Here>';
+CREATE DATABASE aime;
+GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
+FLUSH PRIVILEGES;
+exit;
+```
+
+## Install Python modules
+1. Change your work path to the artemis-master folder using 'cd' and install the requirements:
+
+```shell
+pip install -r requirements.txt
+```	
+
+## Copy/Rename the folder `example_config` to `config`
+
+## Adjust `config/core.yaml`
+
+1. Make sure to change the server `hostname` to be set to your local machine IP (ex.: 192.168.xxx.xxx) 
+    - In case you want to run this only locally, set the following values:
+
+```yaml
+server:
+    listen_address: 0.0.0.0
+title: 
+    hostname: 192.168.xxx.xxx
+```
+
+1. Adjust the proper MySQL information you created earlier
+```yaml
+database:
+  host: "localhost"
+  username: "aime"
+  password: "<Enter Password Here>"
+  name: "aime"
+```
+3. Add the AimeDB key at the bottom of the file
+4. If the webui is needed, change the flag from False to True
+
+## Create the database tables for ARTEMiS
+
+```shell
+python dbutils.py create
+```
+
+## Firewall Adjustements 
+Make sure the following ports are open both on your router and local Windows firewall in case you want to use this for public use (NOT recommended):
+> Port 80 (TCP), 443 (TCP), 8443 (TCP), 22345 (TCP), 8080 (TCP), 8090 (TCP) **webui, 8444 (TCP) **mucha
+
+## Running the ARTEMiS instance
+```shell
+python index.py
+```
+
+# Troubleshooting
+
+## Game does not connect to ARTEMiS Allnet server
+1. Double-check your core.yaml, the listen_address is most likely either not binded to the proper IP or the port is not opened
+
+## Game does not connect to Title Server
+1. Verify that your core.yaml is setup properly for both the server listen_address and title hostname
+2. Boot your game and verify that an AllNet response does show and if it does, attempt to open the URI that is shown under a browser such as Edge, Chrome & Firefox.
+3. If a page is shown, the server is working properly and if it doesn't, double check your port forwarding and also that you have entered the proper local IP under the Title hostname in core.yaml.
+
+## Unhandled command under AimeDB
+1. Double check your AimeDB key under core.yaml, it is incorrect.
+
+## AttributeError: module 'collections' has no attribute 'Hashable'
+1. This means the pyYAML module is obsolete, simply rerun pip with the -U (force update) flag, as shown below.
+    - Change your work path to the artemis-master (or artemis-develop) folder using 'cd' and run the following commands:
+
+```shell
+pip install -r requirements.txt -U
+```

docs/config.md

@@ -1,23 +1,24 @@
 # ARTEMiS Configuration
 ## Server
 - `listen_address`: IP Address or hostname that the server will listen for connections on. Set to 127.0.0.1 for local only, or 0.0.0.0 for all interfaces. Default `127.0.0.1`
+- `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
+- `port`: Port that the server will listen for connections on. Default `80`
+- `ssl_key`: Location of the ssl server key for the secure title server. Ignored if you don't use SSL. Default `cert/title.key`
+- `ssl_cert`: Location of the ssl server certificate for the secure title server. Must not be a self-signed SSL. Ignored if you don't use SSL. Default `cert/title.pem`
 - `allow_user_registration`: Allows users to register in-game via the AimeDB `register` function. Disable to be able to control who can use cards on your server. Default `True`
 - `allow_unregistered_serials`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
 - `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
-- `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
-- `threading`: Flags that `reactor.run` should be called via the `Thread` standard library. May provide a speed boost, but removes the ability to kill the server via `Ctrl + C`. Default: `False`
-- `check_arcade_ip`: Checks IPs against the `arcade` table in the database, if one is defined. Default `False`
-- `strict_ip_checking`: Rejects clients if there is no IP in the `arcade` table for the respective arcade
+- `is_develop`: Flags that the server is a development instance, and enables some useful development features. Disable for production setups. Default `True`.
+- `is_using_proxy`: Flags that you'll be using some other software, such as nginx, to proxy requests, and to send `proxy_port` or `proxy_port_ssl` to games instead of `port`. Default `False`
+- `proxy_port`: Which port your front-facing proxy will be listening on. Ignored if `is_using_proxy` is `False` or if set to `0`. Default `0`
+- `proxy_port`: Which port your front-facing proxy will be listening for ssl connections on. Ignored if `is_using_proxy` is `False` or if set to `0`. Default `0`
 - `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
+- `check_arcade_ip`: Checks IPs against the `arcade` table in the database, if one is defined. Default `False`
+- `strict_ip_checking`: Rejects clients if there is no IP in the `arcade` table for the respective arcade. Default `False`
 ## Title
 - `loglevel`: Logging level for the title server. Default `info`
-- `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
-- `port`: Port that the title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `8080`
-- `port_ssl`: Port that the secure title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `0`
-- `ssl_key`: Location of the ssl server key for the secure title server. Ignored if `port_ssl` is set to `0` or `is_develop` set to `False`. Default `cert/title.key`
-- `ssl_cert`: Location of the ssl server certificate for the secure title server. Must not be a self-signed SSL. Ignored if `port_ssl` is set to `0` or `is_develop` is set to `False`. Default `cert/title.pem`
-- `reboot_start_time`: 24 hour JST time that clients will see as the start of maintenance period. Leave blank for no maintenance time. Default: ""
-- `reboot_end_time`: 24 hour JST time that clients will see as the end of maintenance period. Leave blank for no maintenance time. Default: ""
+- `reboot_start_time`: 24 hour JST time that clients will see as the start of maintenance period, ex `04:00`. Leave blank for no maintenance time. Default: `""`
+- `reboot_end_time`: 24 hour JST time that clients will see as the end of maintenance period, ex `05:00`. Leave blank for no maintenance time. Default: `""`
 ## Database
 - `host`: Host of the database. Default `localhost`
 - `username`: Username of the account the server should connect to the database with. Default `aime`
@@ -25,24 +26,32 @@
 - `name`: Name of the database the server should expect. Default `aime`
 - `port`: Port the database server is listening on. Default `3306`
 - `protocol`: Protocol used in the connection string, e.i `mysql` would result in `mysql://...`. Default `mysql`
-- `sha2_password`: Weather or not the password in the connection string should be hashed via SHA2. Default `False`
-- `loglevel`: Logging level for the database. Default `warn`
-- `user_table_autoincrement_start`: What the `aime_user` table ID autoincrememnt should start with. Default `10000`
+- `sha2_password`: Whether or not the password in the connection string should be hashed via SHA2. Default `False`
+- `loglevel`: Logging level for the database. Default `info`
 - `memcached_host`: Host of the memcached server. Default `localhost`
 ## Frontend
-- `enable`: Weather or not the frontend should be enabled. Default `False`
-- `port`: Port the frontend should listen for connections on. Default `8090`
+- `enable`: Whether or not the frontend servlet should run. Frontend can still be run via `python -m uvicorn core.frontend:app` even if this is set to `False`. Default `False`
+- `port`: Port the frontend should listen on. Default `8080`
 - `loglevel`: Logging level for the frontend server. Default `info`
+- `secret`: Base64-encoded JWT secret for session cookies, generated by you. Default `""`
 ## Allnet
+- `standalone`: Whether allnet should launch it's own servlet on it's own port, or be part of the main servlet on the default port. Disable if you either have something proxying `naominet.jp` requests to port 80, or have port 80 set in `server` -> `port`
+- `port`: Port the allnet server should listen for connections on if it's running standalone. Games are hardcoded to ask for port `80` so only change if you have a proxy redirecting properly. Ignored if `standalone` is `False`. Default `80`
 - `loglevel`: Logging level for the allnet server. Default `info`
-- `port`: Port the allnet server should listen for connections on. Games are hardcoded to ask for port `80` so only change if you have a proxy redirecting properly. Default `80`
 - `allow_online_updates`: Allow allnet to distribute online updates via DownloadOrders. This system is currently non-functional, so leave it disabled. Default `False`
+- `update_cfg_folder`: Folder where delivery INI files will be checked for. Ignored if `allow_online_updates` is `False`. Default `""`
 ## Billing
-- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `8443` so only change if you have a proxy redirecting properly. Set to 0 to use the allnet handler to reduce the number of ports the server eats up. Default `8443`
-- `ssl_key`: Location of the ssl server key for the billing server. Ignored if `port` is set to `0` or `is_develop` set to `False`. Default `cert/server.key`
-- `ssl_cert`: Location of the ssl server certificate for the billing server. Must match the CA distributed to users or the billing server will not connect. Ignored if `port` is set to `0` or `is_develop` is set to `False`. Default `cert/server.pem`
+- `standalone`: Whether the billing server should launch it's own servlet on it's own port, or be part of the main servlet on the default port. Setting this to `True` requires that you have `ssl_key` and `ssl_cert` set. Default `False`
+- `loglevel`: Logging level for the billing server. Default `info`
+- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `8443` so only change if you have a proxy redirecting properly. Ignored if `standalone` is `False`. Default `8443`
+- `ssl_key`: Location of the ssl server key for the billing server. Ignored if `standalone` is `False`. Default `cert/server.key`
+- `ssl_cert`: Location of the ssl server certificate for the billing server. Ignored if `standalone` is `False`.  Must match the CA distributed to users or the billing server will not connect. Default `cert/server.pem`
 - `signing_key`: Location of the RSA Private key used to sign billing requests. Must match the public key distributed to users or the billing server will not connect. Default `cert/billing.key`
 ## Aimedb
+- `enable`: Whether or not aimedb should run. Default `True`
+- `listen_address`: IP Address or hostname that the aimedb server will listen for connections on. Leave this blank to use the listen address under `server`. Default `""`
 - `loglevel`: Logging level for the aimedb server. Default `info`
 - `port`: Port the aimedb server should listen for connections on. Games are hardcoded to ask for port `22345` so only change if you have a proxy redirecting properly. Default `22345`
-- `key`: Key to encrypt/decrypt aimedb requests and responses. MUST be set or the server will not start. If set incorrectly, your server will not properly handle aimedb requests. Default `""`
+- `key`: Key to encrypt/decrypt aimedb requests and responses. MUST be set or the server will not start. If set incorrectly, your server will not properly handle aimedb requests. Default `""`
+- `id_secret`: Base64-encoded JWT secret for Sega Auth IDs. Leaving this blank disables this feature. Default `""`
+- `id_lifetime_seconds`: Number of secons a JWT generated should be valid for. Default `86400` (1 day)

docs/game_specific_info.md

@@ -9,7 +9,15 @@ using the megaime database. Clean installations always create the latest databas
 To upgrade the core database and the database for every game, execute:
 
 ```shell
-python dbutils.py autoupgrade
+python dbutils.py upgrade
+```
+
+If you are using the old master branch that was not setup with alembic, make sure to do the following steps in order:
+- Pull down latest master/develop
+- Update core.yaml
+- Back up your existing database
+```shell
+python dbutils.py migrate
 ```
 
 # Table of content
@@ -22,7 +30,7 @@ python dbutils.py autoupgrade
     - [Card Maker](#card-maker)
     - [WACCA](#wacca)
     - [Sword Art Online Arcade](#sao)
-	- [Initial D THE ARCADE](#initial-d-the-arcade)
+    - [Initial D THE ARCADE](#initial-d-the-arcade)
 
 
 # Supported Games
@@ -62,9 +70,11 @@ Games listed below have been tested and confirmed working.
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --game SDBT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDBT --version <Version ID> --binfolder </path/to/game/data> --optfolder </path/to/game/option/folder>
 ```
 
+**Note: Use the /data not the /bin folder for the Importer!**
+
 The importer for Chunithm will import: Events, Music, Charge Items and Avatar Accesories.
 
 ### Config
@@ -93,7 +103,7 @@ crypto:
 Always make sure your database (tables) are up-to-date:
 
 ```shell
-python dbutils.py --game SDBT upgrade
+python dbutils.py upgrade
 ```
 
 ### Online Battle
@@ -156,7 +166,7 @@ The songId is based on the actual ID within your version of Chunithm.
 In order to use the importer you need to use the provided `Export.csv` file:
 
 ```shell
-python read.py --game SDCA --version <version ID> --binfolder titles/cxb/data
+python read.py --game SDCA --version <Version ID> --binfolder titles/cxb/data
 ```
 
 The importer for crossbeats REV. will import Music.
@@ -192,17 +202,18 @@ Config file is located in `config/cxb.yaml`.
 | SDEZ      | 18         | maimai DX UNiVERSE PLUS |
 | SDEZ      | 19         | maimai DX FESTiVAL      |
 | SDEZ      | 20         | maimai DX FESTiVAL PLUS |
+| SDEZ      | 21         | maimai DX BUDDiES       |
 
 ### Importer
 
 In order to use the importer locate your game installation folder and execute:
 DX:
 ```shell
-python read.py --game <Game Code> --version <Version ID> --binfolder /path/to/StreamingAssets --optfolder /path/to/game/option/folder
+python read.py --game <Game Code> --version <Version ID> --binfolder </path/to/Sinmai_Data> --optfolder </path/to/game/option/folder>
 ```
 Pre-DX:
 ```shell
-python read.py --game <Game Code> --version <Version ID> --binfolder /path/to/data --optfolder /path/to/patch/data
+python read.py --game <Game Code> --version <Version ID> --binfolder </path/to/data> --optfolder </path/to/patch/data>
 ```
 The importer for maimai DX will import Events, Music and Tickets.
 
@@ -215,7 +226,7 @@ The importer for maimai Pre-DX will import Events and Music. Not all games will
 Always make sure your database (tables) are up-to-date:
 
 ```shell
-python dbutils.py --game SDEZ upgrade
+python dbutils.py upgrade
 ```
 
 Pre-Dx uses the same database as DX, so only upgrade using the SDEZ game code!
@@ -235,7 +246,7 @@ Pre-Dx uses the same database as DX, so only upgrade using the SDEZ game code!
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --game SBZV --version <version ID> --binfolder /path/to/game/data/diva --optfolder /path/to/game/data/diva/mdata
+python read.py --game SBZV --version <Version ID> --binfolder </path/to/game/data/diva> --optfolder </path/to/game/data/diva/mdata>
 ```
 
 The importer for Project Diva Arcade will all required data in order to use
@@ -259,7 +270,7 @@ In order to use custom PV Lists, simply drop in your .dat files inside of /title
 Always make sure your database (tables) are up-to-date:
 
 ```shell
-python dbutils.py --game SBZV upgrade
+python dbutils.py upgrade
 ```
 
 ## O.N.G.E.K.I.
@@ -283,7 +294,7 @@ python dbutils.py --game SBZV upgrade
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --game SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDDT --version <Version ID> --binfolder </path/to/game/mu3_Data> --optfolder </path/to/game/option/folder>
 ```
 
 The importer for O.N.G.E.K.I. will all all Cards, Music and Events.
@@ -315,7 +326,7 @@ crypto:
 Always make sure your database (tables) are up-to-date:
 
 ```shell
-python dbutils.py --game SDDT upgrade
+python dbutils.py upgrade
 ```
 
 ### Controlling Events (Ranking Event, Technical Challenge Event, Mission Event)
@@ -406,6 +417,7 @@ After that, on next login the present should be received (or whenever it suppose
   * UNiVERSE PLUS: Yes
   * FESTiVAL: Yes (added in A031)
   * FESTiVAL PLUS: Yes (added in A035)
+  * BUDDiES: Yes (added in A039)
 * O.N.G.E.K.I. bright MEMORY: Yes
 
 
@@ -415,19 +427,19 @@ In order to use the importer you need to use the provided `.csv` files (which ar
 option folders:
 
 ```shell
-python read.py --game SDED --version <version ID> --binfolder titles/cm/cm_data --optfolder /path/to/cardmaker/option/folder
+python read.py --game SDED --version <Version ID> --binfolder titles/cm/cm_data --optfolder </path/to/cardmaker/option/folder>
 ```
 
 **If you haven't already executed the O.N.G.E.K.I. importer, make sure you import all cards!**
 
 ```shell
-python read.py --game SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDDT --version <Version ID> --binfolder </path/to/ongeki/mu3_Data> --optfolder </path/to/ongeki/option/folder>
 ```
 
 Also make sure to import all maimai DX and CHUNITHM data as well:
 
 ```shell
-python read.py --game SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
+python read.py --game SDED --version <Version ID> --binfolder </path/to/cardmaker/CardMaker_Data> --optfolder </path/to/cardmaker/option/folder>
 ```
 
 The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai DX/CHUNITHM) and the hardcoded
@@ -520,7 +532,7 @@ Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded fo
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --game SDFE --version <version ID> --binfolder /path/to/game/WindowsNoEditor/Mercury/Content
+python read.py --game SDFE --version <Version ID> --binfolder </path/to/game/WindowsNoEditor/Mercury/Content>
 ```
 
 The importer for WACCA will import all Music data.
@@ -542,7 +554,7 @@ Config file is located in `config/wacca.yaml`.
 Always make sure your database (tables) are up-to-date:
 
 ```shell
-python dbutils.py --game SDFE upgrade
+python dbutils.py upgrade
 ```
 
 ### VIP Rewards
@@ -594,7 +606,7 @@ Below is a list of VIP rewards. Currently, VIP is not implemented, and thus thes
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --game SDEW --version <version ID> --binfolder /path/to/game/extractedassets
+python read.py --game SDEW --version 0 --binfolder /titles/sao/data/
 ```
 
 The importer for SAO will import all items, heroes, support skills and titles data.
@@ -615,21 +627,23 @@ Config file is located in `config/sao.yaml`.
 Always make sure your database (tables) are up-to-date:
 
 ```shell
-python dbutils.py --game SDEW upgrade
+python dbutils.py upgrade
 ```
 
 ### Notes
 - Defrag Match will crash at loading
 - Co-Op Online is not supported
-- Shop is not functionnal
+- Shop is displayed but cannot purchase heroes or items
 - Player title is currently static and cannot be changed in-game
 - QR Card Scanning currently only load a static hero
-
-**Network hashing in GssSite.dll must be disabled**
+- Ex-quests progression not supported yet
+- Daily Missions not implemented
+- EX TOWER 1,2 & 3 are not yet supported
+- Daily Yui coin not yet fixed
 
 ### Credits for SAO support:
 
-- Midorica - Limited Network Support
+- Midorica - Network Support
 - Dniel97 - Helping with network base
 - tungnotpunk - Source
 
@@ -667,9 +681,9 @@ Config file is located in `config/idac.yaml`.
 | `port_matching`               | Port number for the Online Battle Matching                                                                  |
 | `port_echo1/2`                | Port numbers for Echos                                                                                      |
 | `port_matching_p2p`           | Port number for Online Battle (currently unsupported)                                                       |
-| `stamp.enable`               | Enables/Disabled the play stamp events                                                                      |
+| `stamp.enable`                | Enables/Disabled the play stamp events                                                                      |
 | `stamp.enabled_stamps`        | Define up to 3 play stamp events (without `.json` extension, which are placed in `titles/idac/data/stamps`) |
-| `timetrial.enable`           | Enables/Disables the time trial event                                                                       |
+| `timetrial.enable`            | Enables/Disables the time trial event                                                                       |
 | `timetrial.enabled_timetrial` | Define one! trial event (without `.json` extension, which are placed in `titles/idac/data/timetrial`)       |
 
 
@@ -678,10 +692,11 @@ Config file is located in `config/idac.yaml`.
 Always make sure your database (tables) are up-to-date:
 
 ```shell
-python dbutils.py --game SDGT upgrade
+python dbutils.py upgrade
 ```
 
 ### Notes
+
 - Online Battle is not supported
 - Online Battle Matching is not supported
 
@@ -725,11 +740,10 @@ python dbutils.py --game SDGT upgrade
 
 ### TimeRelease Courses:
 
-
 | Course ID | Course Name               | Direction                |
 | --------- | ------------------------- | ------------------------ |
-| 0         | Akina Lake(秋名湖)        | CounterClockwise(左周り) |
-| 2         | Akina Lake(秋名湖)        | Clockwise(右周り)        |
+| 0         | Lake Akina(秋名湖)        | CounterClockwise(左周り) |
+| 2         | Lake Akina(秋名湖)        | Clockwise(右周り)        |
 | 52        | Hakone(箱根)              | Downhill(下り)           |
 | 54        | Hakone(箱根)              | Hillclimb(上り)          |
 | 36        | Usui(碓氷)                | CounterClockwise(左周り) |
@@ -742,10 +756,10 @@ python dbutils.py --game SDGT upgrade
 | 14        | Akina(秋名)               | Hillclimb(上り)          |
 | 16        | Irohazaka(いろは坂)       | Downhill(下り)           |
 | 18        | Irohazaka(いろは坂)       | Reverse(逆走)            |
-| 56        | Momiji Line(もみじライン) | Downhill(下り)           |
-| 58        | Momiji Line(もみじライン) | Hillclimb(上り)          |
 | 20        | Tsukuba(筑波)             | Outbound(往路)           |
 | 22        | Tsukuba(筑波)             | Inbound(復路)            |
+| 56        | Momiji Line(もみじライン) | Downhill(下り)           |
+| 58        | Momiji Line(もみじライン) | Hillclimb(上り)          |
 | 24        | Happogahara(八方ヶ原)     | Outbound(往路)           |
 | 26        | Happogahara(八方ヶ原)     | Inbound(復路)            |
 | 40        | Sadamine(定峰)            | Downhill(下り)           |
@@ -757,7 +771,67 @@ python dbutils.py --game SDGT upgrade
 | 68        | Odawara(小田原)           | Forward(順走)            |
 | 70        | Odawara(小田原)           | Reverse(逆走)            |
 
-### Credits
+
+### TimeRelease `announce_image`:
+
+- `save_filename`: Filename without file extension saved in the folder `ImageDelivery`
+- `url`: URL to the file on the server with the corresponding file extension (.djg/.gpg) 
+  (except for `display_id=9` where the url is empty)
+- `open_dt`: UNIX timestamp when it should be displayed
+- `close_dt`: UNIX timestamp when it should be hidden
+- `display_id`: One of the following IDS:
+
+| Display ID                               | Description                                                                                       |
+| ---------------------------------------- | ------------------------------------------------------------------------------------------------- |
+| 1                                        | ADV image in the size 1920x1080, shown during attract                                              |
+| 2                                        | Start image in the size 1280x720, shown in the Main Menu after selection the corresponding banner |
+| 3                                        | Banner image in the size 640×120, shown in the Main Menu                                          |
+| 5                                        | Stamp Background image in the size 1780x608                                                       |
+| 6                                        | Online Battle round image in the size 1920x1080                                                   |
+| 8                                        | Stamp Pickup image in the size 624x300, also requires `target_id` set                             |
+| 9                                        | Attract video from the `C:/Mount/Option` folder on real hardware, also requires a `target_id` set |
+
+- `target_id`:
+  - Always 0 unless:
+  - `display_id=8`: Matches an existing stamp pickup abolsute `reward_setting_masu`
+  and will replace the stock image with the provided one from `url`
+  - `display_id=9`: Matches the id from `C:/Mount/Option/MV01/targetXXX.bin`, 
+  where XXX is the `target_id`
+- `page`:
+  - Defines the order in which the images being shown, where 1 is the first image
+  - `display_id` 1, 2, 3: The `page` has to match, so the corresponding images 
+  of an event are shown correctly
+  - `display_id` 7, 8: The `page` defines the `sheet_design` in the play stamps
+- `time`: The time in sec for an image to be shown, always 10
+
+```json
+{
+    "save_filename": "adv_01_example",
+    "url": "http://example.com/images/04721D5D3595FD29778011EC73A8AE77.dpg",
+    "open_dt": 1514761200,
+    "close_dt": 1861916400,
+    "display_id": 1,
+    "target_id": 0,
+    "page": 1,
+    "time": 10
+},
+```
+
+### Battle Gift
+
+- `gift_id`: unique gift index (starts from 0 f.e.)
+- `battle_gift_event_id`: unique event id
+- `reward_category`: item category (f.e. 21 = Chat Stamp)
+- `reward_type`: item id (f.e. 483 = Remilia Scarlet)
+- `reward_name`: name of the reward
+- `rarity`: 2 Golden, 1 Silver, 3 Bronze
+- `cash_rate`: ?
+- `customize_point_rate`: ?
+- `avatar_point_rate`: ?
+- `first_distribution_rate`: only used server side for the probability of the first distribution
+
+### Credits:
+
 - Bottersnike: For the HUGE Reverse Engineering help
 - Kinako: For helping with the timeRelease unlocking of courses and special mode
 

docs/prod.md

@@ -1,41 +1,34 @@
 # ARTEMiS Production mode
-Production mode is a configuration option that changes how the server listens to be more friendly to a production environment. This mode assumes that a proxy (for this guide, nginx) is standing in front of the server to handle port mapping and TLS. In order to activate production mode, simply change `is_develop` to `False` in `core.yaml`. Next time you start the server, you should see "Starting server in production mode".
+ARTEMiS is designed to run in one of two ways. Developmen/local mode, which assumes you're just trying to set up something to save your scores and make the games work, and have patched your games to disable SSL and cert checks and encryption and the like, and production mode. In production mode, artemis assumes you have a proxy server, such as nginx or apache, standing in front of artemis doing HTTPS and port management. This document will cover how to properly set up a production instance of ARTEMiS.
+
+## ARTEMiS configuration
+Step 1 is to edit your artemis configuration. Some recomended changes:
+### `server`
+- `listen_address` -> `127.0.0.1`
+- `is_develop` -> `False`
+- `is_using_proxy` -> `True`
+- `port` -> The port nginx will send proxied requests to. If you're using the example config, set this to 8080.
+- `proxy_port` -> The port your proxy will be accepting title server connections on. If you're using the example config, set this to 80.
+- `proxy_port_ssl` -> The port your proxy will be accepting secure title server connections on. If you're using the example config, set this to 443.
+- `allow_unregistered_serials` -> `False`
+### `billing`
+- `standalone` -> `False`
+### `allnet`
+- `standalone` -> `False`
+### `frontend`
+- `enable` -> `True` if you want the frontend
+- `port` -> `8090` if you're using the default nginx config, otherwise whatever port your proxy will be sending requests to
+### `aimedb`
+- `listen_address` -> `0.0.0.0` unless you're proxying aimedb requests (not recomended at this time), in which case, leave this option unchanged
+
+If you plan to serve artemis behind a VPN, these additional settings are also recomended
+- `check_arcade_ip` -> `True`
+- `strict_ip_checking` -> `True`
 
 ## Nginx Configuration
-### Port forwarding
-Artemis requires that the following ports be forwarded to allow internet traffic to access the server. This will not change regardless of what you set in the config, as many of these ports are hard-coded in the games.
-`tcp:80` all.net, non-ssl titles
-`tcp:8443` billing
-`tcp:22345` aimedb
-`tcp:443` frontend, SSL titles
+For most cases, the config in `example_config` will suffice. It makes the following assumptions
+- ARTEMiS is running on port 8080
+- Billing is set to not be standalone
+- You're not using cloudflare in front of your frontend
 
-### A note about external proxy services (cloudflare, etc)
-Due to the way that artemis functions, it is currently not possible to put the server behind something like Cloudflare. Cloudflare only proxies web traffic on the standard ports (80, 443) and, as shown above, this does not work with artemis. Server administrators should seek other means to protect their network (VPS hosting, VPN, etc)
-
-### SSL Certificates
-You will need to generate SSL certificates for some games. The certificates vary in security and validity requirements. Please see the general guide below
-- General Title: The certificate for the general title server should be valid, not self-signed and match the CN that the game will be reaching out to (e.i if your games are reaching out to titles.hostname.here, your ssl certificate should be valid for titles.hostname.here, or *.hostname.here)
-- CXB: Same requires as the title server. It must not be self-signed, and CN must match. Recomended to get a wildcard cert if possible, and use it for both Title and CXB
-- Pokken: Pokken can be self-signed, and the CN doesn't have to match, but it MUST use 2048-bit RSA. Due to the games age, andthing stronger then that will be rejected.
-
-### Port mappings
-An example config is provided in the `config` folder called `nginx_example.conf`. It is set up for the following:
-`naominet.jp:tcp:80` -> `localhost:tcp:8000` for allnet
-`ib.naominet.jp:ssl:8443` -> `localhost:tcp:8444` for the billing server
-`your.hostname.here:ssl:443` -> `localhost:tcp:8080` for the SSL title server
-`your.hostname.here:tcp:80` -> `localhost:tcp:8080` for the non-SSL title server
-`cxb.hostname.here:ssl:443` -> `localhost:tcp:8080` for crossbeats (appends /SDCA/104/ to the request)
-`pokken.hostname.here:ssl:443` -> `localhost:tcp:8080` for pokken
-`frontend.hostname.here:ssl:443` -> `localhost:tcp:8090` for the frontend, includes https redirection
-
-If you're using this as a guide, be sure to replace your.hostname.here with the hostname you specified in core.yaml under `titles->hostname`. Do *not* change naominet.jp, or allnet/billing will fail. Also remember to specifiy certificate paths correctly, as in the example they are simply placeholders.
-
-### Multi-service ports
-It is possible to use nginx to redirect billing and title server requests to the same port that all.net uses. By setting `port` to 0 under billing and title server, you can change the nginx config to serve the following (entries not shown here should be the same)
-`ib.naominet.jp:ssl:8443` -> `localhost:tcp:8000` for the billing server
-`your.hostname.here:ssl:443` -> `localhost:tcp:8000` for the SSL title server
-`your.hostname.here:tcp:80` -> `localhost:tcp:8000` for the non-SSL title server
-`cxb.hostname.here:ssl:443` -> `localhost:tcp:8000` for crossbeats (appends /SDCA/104/ to the request)
-`pokken.hostname.here:ssl:443` -> `localhost:tcp:8000` for pokken
-
-This will allow you to only use 3 ports locally, but you will still need to forward the same internet-facing ports as before.
+If this describes you, your only configuration needs are to edit the `server_name` and `certificate_*` directives. Otherwise, please see nginx configuration documentation to configure it to best suit your setup.

example_config/core.yaml

@@ -1,26 +1,25 @@
 server:
-  listen_address: "127.0.0.1"
+  listen_address: "127.0.0.1"  
+  hostname: "localhost"  
+  port: 80
+  ssl_key: "cert/title.key"
+  ssl_cert: "cert/title.crt"
   allow_user_registration: True
   allow_unregistered_serials: True
   name: "ARTEMiS"
   is_develop: True
   is_using_proxy: False
-  threading: False
+  proxy_port: 0
+  proxy_port_ssl: 0
   log_dir: "logs"
   check_arcade_ip: False
   strict_ip_checking: False
 
 title:
   loglevel: "info"
-  hostname: "localhost"
-  port: 8080
-  port_ssl: 0
-  ssl_cert: "cert/title.crt"
-  ssl_key: "cert/title.key"
   reboot_start_time: "04:00"
   reboot_end_time: "05:00"
 
-
 database:
   host: "localhost"
   username: "aime"
@@ -29,30 +28,34 @@ database:
   port: 3306
   protocol: "mysql"
   sha2_password: False
-  loglevel: "warn"
-  user_table_autoincrement_start: 10000
+  loglevel: "info"
   enable_memcached: True
   memcached_host: "localhost"
 
 frontend:
   enable: False
-  port: 8090
+  port: 8080
   loglevel: "info"
+  secret: ""
 
 allnet:
-  loglevel: "info"
+  standalone: False
   port: 80
-  ip_check: False
+  loglevel: "info"
   allow_online_updates: False
   update_cfg_folder: ""
 
 billing:
+  standalone: True
+  loglevel: "info"
   port: 8443
   ssl_key: "cert/server.key"
   ssl_cert: "cert/server.pem"
   signing_key: "cert/billing.key"
 
 aimedb:
+  enable: True
+  listen_address: ""
   loglevel: "info"
   port: 22345
   key: ""
@@ -60,6 +63,4 @@ aimedb:
   id_lifetime_seconds: 86400
 
 mucha:
-  enable: False
-  hostname: "localhost"
   loglevel: "info"

example_config/cxb.yaml

@@ -1,3 +1,4 @@
 server:
   enable: True
-  loglevel: "info"
+  loglevel: "info"
+  use:https: True

example_config/idac.yaml

@@ -10,6 +10,10 @@ server:
   port_echo2: 20002
   port_matching_p2p: 20003
 
+timerelease:
+  timerelease_no: 3
+  timerelease_avatar_gacha_no: 3
+
 stamp:
   enable: True
   enabled_stamps: # max 3 play stamps
@@ -20,3 +24,7 @@ stamp:
 timetrial:
   enable: True
   enabled_timetrial: "touhou_remilia_scarlet"
+
+battle_event:
+  enabled: True
+  enabled_battle_event: "touhou_1st"

example_config/nginx_example.conf

@@ -6,7 +6,7 @@ server {
 	location / {
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_pass_request_headers on;
-		proxy_pass http://localhost:8000/;
+		proxy_pass http://127.0.0.1:8080/;
 	}
 }
 
@@ -18,7 +18,7 @@ server {
 	location / {
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_pass_request_headers on;
-		proxy_pass http://localhost:8080/;
+		proxy_pass http://127.0.0.1:8080/;
 	}
 }
 
@@ -38,11 +38,13 @@ server {
 	ssl_prefer_server_ciphers off;
 
 	location / {
-		proxy_pass http://localhost:8080/;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_pass_request_headers on;
+		proxy_pass http://127.0.0.1:8080/;
 	}
 }
 
-# Billing
+# Billing, comment this out if running billing standalone
 server {
 	listen 8443 ssl;	
 	server_name ib.naominet.jp;
@@ -57,30 +59,10 @@ server {
 	ssl_ciphers "ALL:@SECLEVEL=0";
 	ssl_prefer_server_ciphers off;
 
-	location / {
-		proxy_pass http://localhost:8444/;
-	}
-}
-
-# Pokken, comment this out if you don't plan on serving pokken.
-server {
-	listen 443 ssl;	
-	server_name pokken.hostname.here;
-	
-	ssl_certificate /path/to/cert/pokken.pem;
-	ssl_certificate_key /path/to/cert/pokken.key;
-	ssl_session_timeout 1d;
-	ssl_session_cache shared:MozSSL:10m;
-	ssl_session_tickets off;
-
-	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
-	ssl_ciphers "ALL:@SECLEVEL=0";
-	ssl_prefer_server_ciphers off;
-
 	location / {
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_pass_request_headers on;
-		proxy_pass http://localhost:8080/;
+		proxy_pass http://127.0.0.1:8080/;
 	}
 }
 
@@ -91,12 +73,12 @@ server {
 
     location / {
         return 301 https://$host$request_uri;
-		# If you don't want https redirection, comment the line above and uncomment the line below
-		# proxy_pass http://localhost:8090/;
+		# If you don't want https redirection, or are using something like cloudflare to manage HTTPS, comment out the line above and uncomment the line below
+		# proxy_pass http://127.0.0.1:8090/;
     }
 }
 
-# Frontend HTTPS. Comment out if you on't intend to use the frontend
+# Frontend HTTPS. Comment out if you on't intend to use the frontend, or have cloudflare or something managing https for you.
 server {
     listen 443 ssl;
 	server_name frontend.hostname.here;
@@ -118,6 +100,6 @@ server {
 	location / {
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_pass_request_headers on;
-		proxy_pass http://localhost:8090/;
+		proxy_pass http://127.0.0.1:8090/;
 	}
 }

index.py

@@ -1,335 +1,117 @@
 #!/usr/bin/env python3
 import argparse
-import logging, coloredlogs
-from logging.handlers import TimedRotatingFileHandler
-from typing import Dict
 import yaml
-from os import path, mkdir, access, W_OK
-from core import *
+from os import path, environ
+import uvicorn
+import logging
+import asyncio
 
-from twisted.web import server, resource
-from twisted.internet import reactor, endpoints
-from twisted.web.http import Request
-from routes import Mapper
-from threading import Thread
+from core import CoreConfig, AimedbServlette
 
-class HttpDispatcher(resource.Resource):
-    def __init__(self, cfg: CoreConfig, config_dir: str):
-        super().__init__()
-        self.config = cfg
-        self.isLeaf = True
-        self.map_get = Mapper()
-        self.map_post = Mapper()
-        self.logger = logging.getLogger("core")
-
-        self.title = TitleServlet(cfg, config_dir)
-        self.allnet = AllnetServlet(cfg, config_dir)
-        self.mucha = MuchaServlet(cfg, config_dir)
-
-        self.map_get.connect(
-            "allnet_downloadorder_ini",
-            "/dl/ini/{file}",
-            controller="allnet",
-            action="handle_dlorder_ini",
-            conditions=dict(method=["GET"]),
+async def launch_main(cfg: CoreConfig, ssl: bool) -> None:
+    if ssl:
+        server_cfg = uvicorn.Config(
+            "core.app:app", 
+            host=cfg.server.listen_address, 
+            port=cfg.server.port if args.port == 0 else args.port, 
+            reload=cfg.server.is_develop,
+            log_level="info" if cfg.server.is_develop else "critical",
+            ssl_version=3,
+            ssl_certfile=cfg.server.ssl_cert,
+            ssl_keyfile=cfg.server.ssl_key
+        ) 
+    else:
+        server_cfg = uvicorn.Config(
+            "core.app:app", 
+            host=cfg.server.listen_address, 
+            port=cfg.server.port if args.port == 0 else args.port, 
+            reload=cfg.server.is_develop,
+            log_level="info" if cfg.server.is_develop else "critical"
         )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
 
-        self.map_post.connect(
-            "allnet_downloadorder_report",
-            "/report-api/Report",
-            controller="allnet",
-            action="handle_dlorder_report",
-            conditions=dict(method=["POST"]),
-        )
+async def launch_billing(cfg: CoreConfig) -> None:
+    server_cfg = uvicorn.Config(
+        "core.allnet:app_billing", 
+        host=cfg.server.listen_address, 
+        port=cfg.billing.port, 
+        reload=cfg.server.is_develop,
+        log_level="info" if cfg.server.is_develop else "critical",
+        ssl_version=3,
+        ssl_certfile=cfg.billing.ssl_cert,
+        ssl_keyfile=cfg.billing.ssl_key,
+        ssl_ciphers="DEFAULT:!aNULL:!eNULL:!MD5:!3DES:!DES:!RC4:!IDEA:!SEED:!aDSS:!SRP:!PSK",
+    )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
 
-        self.map_get.connect(
-            "allnet_ping",
-            "/naomitest.html",
-            controller="allnet",
-            action="handle_naomitest",
-            conditions=dict(method=["GET"]),
-        )
-        self.map_post.connect(
-            "allnet_poweron",
-            "/sys/servlet/PowerOn",
-            controller="allnet",
-            action="handle_poweron",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_downloadorder",
-            "/sys/servlet/DownloadOrder",
-            controller="allnet",
-            action="handle_dlorder",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_loaderstaterecorder",
-            "/sys/servlet/LoaderStateRecorder",
-            controller="allnet",
-            action="handle_loaderstaterecorder",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_alive",
-            "/sys/servlet/Alive",
-            controller="allnet",
-            action="handle_alive",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_get.connect(
-            "allnet_alive",
-            "/sys/servlet/Alive",
-            controller="allnet",
-            action="handle_alive",
-            conditions=dict(method=["GET"]),
-        )
-        self.map_post.connect(
-            "allnet_billing",
-            "/request",
-            controller="allnet",
-            action="handle_billing_request",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_billing",
-            "/request/",
-            controller="allnet",
-            action="handle_billing_request",
-            conditions=dict(method=["POST"]),
-        )
+async def launch_frontend(cfg: CoreConfig) -> None:
+    server_cfg = uvicorn.Config(
+        "core.frontend:app", 
+        host=cfg.server.listen_address, 
+        port=cfg.frontend.port, 
+        reload=cfg.server.is_develop,
+        log_level="info" if cfg.server.is_develop else "critical",
+    )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
 
-        # Maintain compatability
-        self.map_post.connect(
-            "mucha_boardauth",
-            "/mucha/boardauth.do",
-            controller="mucha",
-            action="handle_boardauth",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_updatacheck",
-            "/mucha/updatacheck.do",
-            controller="mucha",
-            action="handle_updatecheck",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_dlstate",
-            "/mucha/downloadstate.do",
-            controller="mucha",
-            action="handle_dlstate",
-            conditions=dict(method=["POST"]),
-        )
+async def launch_allnet(cfg: CoreConfig) -> None:
+    server_cfg = uvicorn.Config(
+        "core.allnet:app_allnet", 
+        host=cfg.server.listen_address, 
+        port=cfg.allnet.port, 
+        reload=cfg.server.is_develop,
+        log_level="info" if cfg.server.is_develop else "critical",
+    )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
 
-        self.map_post.connect(
-            "mucha_boardauth",
-            "/mucha_front/boardauth.do",
-            controller="mucha",
-            action="handle_boardauth",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_updatacheck",
-            "/mucha_front/updatacheck.do",
-            controller="mucha",
-            action="handle_updatecheck",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_dlstate",
-            "/mucha_front/downloadstate.do",
-            controller="mucha",
-            action="handle_dlstate",
-            conditions=dict(method=["POST"]),
-        )
-
-        for code, game in self.title.title_registry.items():
-            get_matchers, post_matchers = game.get_endpoint_matchers()
-            
-            for m in get_matchers:
-                self.map_get.connect(
-                    "title_get",
-                    m[1],
-                    controller="title",
-                    action="render_GET",
-                    title=code,
-                    subaction=m[0],
-                    conditions=dict(method=["GET"]),
-                    requirements=m[2],
-                )
-                
-            for m in post_matchers:
-                self.map_post.connect(
-                    "title_post",
-                    m[1],
-                    controller="title",
-                    action="render_POST",
-                    title=code,
-                    subaction=m[0],
-                    conditions=dict(method=["POST"]),
-                    requirements=m[2],
-                )
-
-    def render_GET(self, request: Request) -> bytes:
-        test = self.map_get.match(request.uri.decode())
-        client_ip = Utils.get_ip_addr(request)
-
-        if test is None:
-            self.logger.debug(
-                f"Unknown GET endpoint {request.uri.decode()} from {client_ip} to port {request.getHost().port}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        return self.dispatch(test, request)
-
-    def render_POST(self, request: Request) -> bytes:
-        test = self.map_post.match(request.uri.decode())
-        client_ip = Utils.get_ip_addr(request)
-
-        if test is None:
-            self.logger.debug(
-                f"Unknown POST endpoint {request.uri.decode()} from {client_ip} to port {request.getHost().port}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        return self.dispatch(test, request)
-
-    def dispatch(self, matcher: Dict, request: Request) -> bytes:
-        controller = getattr(self, matcher["controller"], None)
-        if controller is None:
-            self.logger.error(
-                f"Controller {matcher['controller']} not found via endpoint {request.uri.decode()}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        handler = getattr(controller, matcher["action"], None)
-        if handler is None:
-            self.logger.error(
-                f"Action {matcher['action']} not found in controller {matcher['controller']} via endpoint {request.uri.decode()}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        url_vars = matcher
-        url_vars.pop("controller")
-        url_vars.pop("action")
-        ret = handler(request, url_vars)
-
-        if type(ret) == str:
-            return ret.encode()
-        
-        elif type(ret) == bytes or type(ret) == tuple: # allow for bytes or tuple (data, response code) responses
-            return ret
-        
-        elif ret is None:
-            self.logger.warning(f"None returned by controller for {request.uri.decode()} endpoint")
-            return b""
-        
-        else:
-            self.logger.warning(f"Unknown data type returned by controller for {request.uri.decode()} endpoint")
-            return b""
 
+async def launcher(cfg: CoreConfig, ssl: bool) -> None:
+    task_list = [asyncio.create_task(launch_main(cfg, ssl))]
+    
+    if cfg.billing.standalone:
+        task_list.append(asyncio.create_task(launch_billing(cfg)))
+    if cfg.frontend.enable:
+        task_list.append(asyncio.create_task(launch_frontend(cfg)))
+    if cfg.allnet.standalone:
+        task_list.append(asyncio.create_task(launch_allnet(cfg)))
+    if cfg.aimedb.enable:
+        AimedbServlette(cfg).start()
+    
+    done, pending = await asyncio.wait(
+        task_list,
+        return_when=asyncio.FIRST_COMPLETED,
+    )
+    
+    logging.getLogger("core").info("Shutdown")
+    for pending_task in pending:
+        pending_task.cancel("Another service died, server is shutting down")
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="ARTEMiS main entry point")
+    parser = argparse.ArgumentParser(description="Artemis main entry point")
     parser.add_argument(
         "--config", "-c", type=str, default="config", help="Configuration folder"
     )
+    parser.add_argument(
+        "--port", "-p", type=int, default=0, help="Port override"
+    )
+    parser.add_argument(
+        "--ssl", "-s", type=bool, help="Launch with SSL"
+    )
     args = parser.parse_args()
 
     if not path.exists(f"{args.config}/core.yaml"):
         print(
-            f"The config folder you specified ({args.config}) does not exist or does not contain core.yaml.\nDid you copy the example folder?"
+            f"The config folder you specified ({args.config}) does not exist or does not contain core.yaml. Defaults will be used.\nDid you copy the example folder?"
         )
-        exit(1)
-
+    
     cfg: CoreConfig = CoreConfig()
     if path.exists(f"{args.config}/core.yaml"):
         cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
 
-    if not path.exists(cfg.server.log_dir):
-        mkdir(cfg.server.log_dir)
+    environ["ARTEMIS_CFG_DIR"] = args.config
 
-    if not access(cfg.server.log_dir, W_OK):
-        print(
-            f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
-        )
-        exit(1)
-
-    logger = logging.getLogger("core")
-    log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
-    log_fmt = logging.Formatter(log_fmt_str)
-
-    fileHandler = TimedRotatingFileHandler(
-        "{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10
-    )
-    fileHandler.setFormatter(log_fmt)
-
-    consoleHandler = logging.StreamHandler()
-    consoleHandler.setFormatter(log_fmt)
-
-    logger.addHandler(fileHandler)
-    logger.addHandler(consoleHandler)
-
-    log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
-    logger.setLevel(log_lv)
-    coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
-
-    if not cfg.aimedb.key:
-        logger.error("!!AIMEDB KEY BLANK, SET KEY IN CORE.YAML!!")
-        exit(1)
-
-    logger.info(
-        f"ARTEMiS starting in {'develop' if cfg.server.is_develop else 'production'} mode"
-    )
-
-    allnet_server_str = f"tcp:{cfg.allnet.port}:interface={cfg.server.listen_address}"
-    title_server_str = f"tcp:{cfg.title.port}:interface={cfg.server.listen_address}"
-    title_https_server_str = f"ssl:{cfg.title.port_ssl}:interface={cfg.server.listen_address}:privateKey={cfg.title.ssl_key}:certKey={cfg.title.ssl_cert}"
-    adb_server_str = f"tcp:{cfg.aimedb.port}:interface={cfg.server.listen_address}"
-    frontend_server_str = (
-        f"tcp:{cfg.frontend.port}:interface={cfg.server.listen_address}"
-    )
-
-    billing_server_str = f"tcp:{cfg.billing.port}:interface={cfg.server.listen_address}"
-    if cfg.server.is_develop:
-        billing_server_str = (
-            f"ssl:{cfg.billing.port}:interface={cfg.server.listen_address}"
-            f":privateKey={cfg.billing.ssl_key}:certKey={cfg.billing.ssl_cert}"
-        )
-
-    dispatcher = HttpDispatcher(cfg, args.config)
-
-    endpoints.serverFromString(reactor, allnet_server_str).listen(
-        server.Site(dispatcher)
-    )
-    endpoints.serverFromString(reactor, adb_server_str).listen(AimedbFactory(cfg))
-
-    if cfg.frontend.enable:
-        endpoints.serverFromString(reactor, frontend_server_str).listen(
-            server.Site(FrontendServlet(cfg, args.config))
-        )
-
-    if cfg.billing.port > 0:
-        endpoints.serverFromString(reactor, billing_server_str).listen(
-            server.Site(dispatcher)
-        )
-
-    if cfg.title.port > 0:
-        endpoints.serverFromString(reactor, title_server_str).listen(
-            server.Site(dispatcher)
-        )
-    
-    if cfg.title.port_ssl > 0:
-        endpoints.serverFromString(reactor, title_https_server_str).listen(
-            server.Site(dispatcher)
-        )
-
-    if cfg.server.threading:
-        Thread(target=reactor.run, args=(False,)).start()
-    else:
-        reactor.run()
+    asyncio.run(launcher(cfg, args.ssl))

read.py

@@ -1,4 +1,4 @@
-# vim: set fileencoding=utf-8
+#!/usr/bin/env python3
 import argparse
 import re
 import os
@@ -6,6 +6,7 @@ import yaml
 from os import path
 import logging
 import coloredlogs
+import asyncio
 
 from logging.handlers import TimedRotatingFileHandler
 from typing import List, Optional
@@ -38,6 +39,9 @@ class BaseReader:
                     ret.append(f"{root}/{dir}")
 
         return ret
+    
+    async def read(self) -> None:
+        pass
 
 
 if __name__ == "__main__":
@@ -136,6 +140,8 @@ if __name__ == "__main__":
     for dir, mod in titles.items():
         if args.game in mod.game_codes:
             handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra)
-            handler.read()
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(handler.read())
+            
 
     logger.info("Done")

readme.md

@@ -4,37 +4,68 @@ A network service emulator for games running SEGA'S ALL.NET service, and similar
 # Supported games
 Games listed below have been tested and confirmed working. Only game versions older then the version currently active in arcades, or games versions that have not recieved a major update in over one year, are supported.
 
-+ CHUNITHM
-    + All versions up to SUN PLUS
-
-+ crossbeats REV.
-    + All versions + omnimix
-
-+ maimai DX
-    + All versions up to FESTiVAL PLUS
-
-+ Hatsune Miku: Project DIVA Arcade
-    + All versions
-
 + Card Maker
     + 1.30
     + 1.35
 
-+ O.N.G.E.K.I.
-    + All versions up to bright MEMORY
++ CHUNITHM INTL
+    + SUPERSTAR
+    + NEW
+    + NEW PLUS
+    + SUN
+    + SUN PLUS
 
-+ WACCA
-    + Lily R
-    + Reverse
++ CHUNITHM JP
+    + AIR
+    + AIR PLUS
+    + AMAZON
+    + AMAZON PLUS
+    + CRYSTAL
+    + CRYSTAL PLUS
+    + PARADISE
+    + PARADISE LOST
+    + NEW
+    + NEW PLUS
+    + SUN
+    + SUN PLUS
+
++ crossbeats REV.
+    + Crossbeats REV.
+    + Crossbeats REV. SUNRiSE S1
+    + Crossbeats REV. SUNRiSE S2 + omnimix
+
++ Hatsune Miku: Project DIVA Arcade
+    + Future Tone Arcade - All versions
+
++ Initial D THE ARCADE
+    + Season 2
+
++ maimai DX
+    + Splash
+    + Splash Plus
+    + UNiVERSE
+    + UNiVERSE PLUS
+    + FESTiVAL
+    + FESTiVAL PLUS
+    + BUDDiES
+
++ O.N.G.E.K.I.
+    + SUMMER
+    + SUMMER PLUS
+    + R.E.D.
+    + R.E.D. PLUS
+    + bright
+    + bright MEMORY
 
 + POKKÉN TOURNAMENT
     + Final Online
 
-+ Sword Art Online Arcade (partial support)
-    + Final
++ Sword Art Online Arcade
+    + Final (Single player only)
 
-+ Initial D THE ARCADE
-    + Season 2
++ WACCA
+    + Lily R
+    + Reverse
 
 ## Requirements
 - python 3 (tested working with 3.9 and 3.10, other versions YMMV)
@@ -43,7 +74,7 @@ Games listed below have been tested and confirmed working. Only game versions ol
 - mysql/mariadb server
 
 ## Setup guides
-Follow the platform-specific guides for [windows](docs/INSTALL_WINDOWS.md), [ubuntu](docs/INSTALL_UBUNTU.md) or [docker](docs/INSTALL_DOCKER.md) to setup and run the server.
+Follow the platform-specific guides for [windows](docs/INSTALL_WINDOWS.md), [linux (Debian 12 or Rasperry Pi OS recomended, but anything works)](docs/INSTALL_LINUX.md) or [docker](docs/INSTALL_DOCKER.md) to setup and run the server.
 
 ## Game specific information
 Read [Games specific info](docs/game_specific_info.md) for all supported games, importer settings, configuration option and database upgrades.

requirements.txt

@@ -1,21 +1,24 @@
-mypy
-wheel
-twisted
-pytz
-pyyaml
-sqlalchemy==1.4.46
-mysqlclient
-pyopenssl
-service_identity
-PyCryptodome
-inflection
-coloredlogs
-pylibmc; platform_system != "Windows"
-wacky
-Routes
-bcrypt
-jinja2
-protobuf
-autobahn
-pillow
-pyjwt
+mypy
+wheel
+pytz
+pyyaml
+sqlalchemy==1.4.46
+mysqlclient
+pyopenssl
+service_identity
+PyCryptodome
+inflection
+coloredlogs
+pylibmc; platform_system != "Windows"
+wacky
+bcrypt
+jinja2
+protobuf
+pillow
+pyjwt==2.8.0
+websockets
+starlette
+asyncio
+uvicorn
+alembic
+python-multipart

titles/chuni/__init__.py

@@ -1,10 +1,11 @@
-from titles.chuni.index import ChuniServlet
-from titles.chuni.const import ChuniConstants
-from titles.chuni.database import ChuniData
-from titles.chuni.read import ChuniReader
+from .index import ChuniServlet
+from .const import ChuniConstants
+from .database import ChuniData
+from .read import ChuniReader
+from .frontend import ChuniFrontend
 
 index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
+frontend = ChuniFrontend
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT]
-current_schema_version = 5

titles/chuni/air.py

@@ -11,7 +11,7 @@ class ChuniAir(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AIR
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.10.00"
         return ret

titles/chuni/airplus.py

@@ -11,7 +11,7 @@ class ChuniAirPlus(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AIR_PLUS
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.15.00"
         return ret

titles/chuni/amazon.py

@@ -13,7 +13,7 @@ class ChuniAmazon(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AMAZON
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.30.00"
         return ret

titles/chuni/amazonplus.py

@@ -13,7 +13,7 @@ class ChuniAmazonPlus(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
         return ret

titles/chuni/base.py

@@ -22,7 +22,7 @@ class ChuniBase:
         self.game = ChuniConstants.GAME_CODE
         self.version = ChuniConstants.VER_CHUNITHM
 
-    def handle_game_login_api_request(self, data: Dict) -> Dict:
+    async def handle_game_login_api_request(self, data: Dict) -> Dict:
         """
         Handles the login bonus logic, required for the game because
         getUserLoginBonus gets called after getUserItem and therefore the
@@ -38,20 +38,20 @@ class ChuniBase:
             return {"returnCode": 1}
 
         user_id = data["userId"]
-        login_bonus_presets = self.data.static.get_login_bonus_presets(self.version)
+        login_bonus_presets = await self.data.static.get_login_bonus_presets(self.version)
 
         for preset in login_bonus_presets:
             # check if a user already has some pogress and if not add the
             # login bonus entry
-            user_login_bonus = self.data.item.get_login_bonus(
+            user_login_bonus = await self.data.item.get_login_bonus(
                 user_id, self.version, preset["presetId"]
             )
             if user_login_bonus is None:
-                self.data.item.put_login_bonus(
+                await self.data.item.put_login_bonus(
                     user_id, self.version, preset["presetId"]
                 )
                 # yeah i'm lazy
-                user_login_bonus = self.data.item.get_login_bonus(
+                user_login_bonus = await self.data.item.get_login_bonus(
                     user_id, self.version, preset["presetId"]
                 )
 
@@ -67,7 +67,7 @@ class ChuniBase:
                 bonus_count = user_login_bonus["bonusCount"] + 1
                 last_update_date = datetime.now()
 
-                all_login_boni = self.data.static.get_login_bonus(
+                all_login_boni = await self.data.static.get_login_bonus(
                     self.version, preset["presetId"]
                 )
 
@@ -91,13 +91,13 @@ class ChuniBase:
                         is_finished = True
 
                 # grab the item for the corresponding day
-                login_item = self.data.static.get_login_bonus_by_required_days(
+                login_item = await self.data.static.get_login_bonus_by_required_days(
                     self.version, preset["presetId"], bonus_count
                 )
                 if login_item is not None:
                     # now add the present to the database so the
                     # handle_get_user_item_api_request can grab them
-                    self.data.item.put_item(
+                    await self.data.item.put_item(
                         user_id,
                         {
                             "itemId": login_item["presentId"],
@@ -107,7 +107,7 @@ class ChuniBase:
                         },
                     )
 
-                self.data.item.put_login_bonus(
+                await self.data.item.put_login_bonus(
                     user_id,
                     self.version,
                     preset["presetId"],
@@ -119,12 +119,12 @@ class ChuniBase:
 
         return {"returnCode": 1}
 
-    def handle_game_logout_api_request(self, data: Dict) -> Dict:
+    async def handle_game_logout_api_request(self, data: Dict) -> Dict:
         # self.data.base.log_event("chuni", "logout", logging.INFO, {"version": self.version, "user": data["userId"]})
         return {"returnCode": 1}
 
-    def handle_get_game_charge_api_request(self, data: Dict) -> Dict:
-        game_charge_list = self.data.static.get_enabled_charges(self.version)
+    async def handle_get_game_charge_api_request(self, data: Dict) -> Dict:
+        game_charge_list = await self.data.static.get_enabled_charges(self.version)
 
         if game_charge_list is None or len(game_charge_list) == 0:
             return {"length": 0, "gameChargeList": []}
@@ -145,8 +145,8 @@ class ChuniBase:
             )
         return {"length": len(charges), "gameChargeList": charges}
 
-    def handle_get_game_event_api_request(self, data: Dict) -> Dict:
-        game_events = self.data.static.get_enabled_events(self.version)
+    async def handle_get_game_event_api_request(self, data: Dict) -> Dict:
+        game_events = await self.data.static.get_enabled_events(self.version)
 
         if game_events is None or len(game_events) == 0:
             self.logger.warning("No enabled events, did you run the reader?")
@@ -177,10 +177,10 @@ class ChuniBase:
             "gameEventList": event_list,
         }
 
-    def handle_get_game_idlist_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_idlist_api_request(self, data: Dict) -> Dict:
         return {"type": data["type"], "length": 0, "gameIdlistList": []}
 
-    def handle_get_game_message_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_message_api_request(self, data: Dict) -> Dict:
         return {
             "type": data["type"], 
             "length": 1, 
@@ -193,14 +193,14 @@ class ChuniBase:
             }]
         }
 
-    def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
-        rankings = self.data.score.get_rankings(self.version)
+    async def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
+        rankings = await self.data.score.get_rankings(self.version)
         return {"type": data["type"], "gameRankingList": rankings}
 
-    def handle_get_game_sale_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_sale_api_request(self, data: Dict) -> Dict:
         return {"type": data["type"], "length": 0, "gameSaleList": []}
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         # if reboot start/end time is not defined use the default behavior of being a few hours ago
         if self.core_cfg.title.reboot_start_time == "" or self.core_cfg.title.reboot_end_time == "":
             reboot_start = datetime.strftime(
@@ -240,8 +240,8 @@ class ChuniBase:
             "isDumpUpload": "false",
             "isAou": "false",
         }
-    def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
-        user_activity_list = self.data.profile.get_profile_activity(
+    async def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
+        user_activity_list = await self.data.profile.get_profile_activity(
             data["userId"], data["kind"]
         )
 
@@ -261,8 +261,8 @@ class ChuniBase:
             "userActivityList": activity_list,
         }
 
-    def handle_get_user_character_api_request(self, data: Dict) -> Dict:
-        characters = self.data.item.get_characters(data["userId"])
+    async def handle_get_user_character_api_request(self, data: Dict) -> Dict:
+        characters = await self.data.item.get_characters(data["userId"])
         if characters is None:
             return {
                 "userId": data["userId"],
@@ -296,8 +296,8 @@ class ChuniBase:
             "userCharacterList": character_list,
         }
 
-    def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
-        user_charge_list = self.data.profile.get_profile_charge(data["userId"])
+    async def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
+        user_charge_list = await self.data.profile.get_profile_charge(data["userId"])
 
         charge_list = []
         for charge in user_charge_list:
@@ -312,15 +312,15 @@ class ChuniBase:
             "userChargeList": charge_list,
         }
 
-    def handle_get_user_recent_player_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_recent_player_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
             "userRecentPlayerList": [], # playUserId, playUserName, playDate, friendPoint
         }
 
-    def handle_get_user_course_api_request(self, data: Dict) -> Dict:
-        user_course_list = self.data.score.get_courses(data["userId"])
+    async def handle_get_user_course_api_request(self, data: Dict) -> Dict:
+        user_course_list = await self.data.score.get_courses(data["userId"])
         if user_course_list is None:
             return {
                 "userId": data["userId"],
@@ -354,8 +354,8 @@ class ChuniBase:
             "userCourseList": course_list,
         }
 
-    def handle_get_user_data_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_data(data["userId"], self.version)
+    async def handle_get_user_data_api_request(self, data: Dict) -> Dict:
+        p = await self.data.profile.get_profile_data(data["userId"], self.version)
         if p is None:
             return {}
 
@@ -366,8 +366,8 @@ class ChuniBase:
 
         return {"userId": data["userId"], "userData": profile}
 
-    def handle_get_user_data_ex_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_data_ex(data["userId"], self.version)
+    async def handle_get_user_data_ex_api_request(self, data: Dict) -> Dict:
+        p = await self.data.profile.get_profile_data_ex(data["userId"], self.version)
         if p is None:
             return {}
 
@@ -378,8 +378,8 @@ class ChuniBase:
 
         return {"userId": data["userId"], "userDataEx": profile}
 
-    def handle_get_user_duel_api_request(self, data: Dict) -> Dict:
-        user_duel_list = self.data.item.get_duels(data["userId"])
+    async def handle_get_user_duel_api_request(self, data: Dict) -> Dict:
+        user_duel_list = await self.data.item.get_duels(data["userId"])
         if user_duel_list is None:
             return {}
 
@@ -396,8 +396,8 @@ class ChuniBase:
             "userDuelList": duel_list,
         }
 
-    def handle_get_user_rival_data_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_rival(data["rivalId"])
+    async def handle_get_user_rival_data_api_request(self, data: Dict) -> Dict:
+        p = await self.data.profile.get_rival(data["rivalId"])
         if p is None:
             return {}
         userRivalData = {
@@ -409,14 +409,14 @@ class ChuniBase:
             "userRivalData": userRivalData
         }
     
-    def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
         rival_id = data["rivalId"]
         next_index = int(data["nextIndex"])
         max_count = int(data["maxCount"])
         user_rival_music_list = []
 
         # Fetch all the rival music entries for the user
-        all_entries = self.data.score.get_rival_music(rival_id)
+        all_entries = await self.data.score.get_rival_music(rival_id)
 
         # Process the entries based on max_count and nextIndex
         for music in all_entries:
@@ -462,12 +462,12 @@ class ChuniBase:
         return result
 
     
-    def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
         user_fav_item_list = []
 
         # still needs to be implemented on WebUI
         # 1: Music, 2: User, 3: Character
-        fav_list = self.data.item.get_all_favorites(
+        fav_list = await self.data.item.get_all_favorites(
             data["userId"], self.version, fav_kind=int(data["kind"])
         )
         if fav_list is not None:
@@ -482,17 +482,17 @@ class ChuniBase:
             "userFavoriteItemList": user_fav_item_list,
         }
 
-    def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
         """
         This is handled via the webui, which we don't have right now
         """
 
         return {"userId": data["userId"], "length": 0, "userFavoriteMusicList": []}
 
-    def handle_get_user_item_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
         kind = int(int(data["nextIndex"]) / 10000000000)
         next_idx = int(int(data["nextIndex"]) % 10000000000)
-        user_item_list = self.data.item.get_items(data["userId"], kind)
+        user_item_list = await self.data.item.get_items(data["userId"], kind)
 
         if user_item_list is None or len(user_item_list) == 0:
             return {
@@ -526,9 +526,9 @@ class ChuniBase:
             "userItemList": items,
         }
 
-    def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
         user_id = data["userId"]
-        user_login_bonus = self.data.item.get_all_login_bonus(user_id, self.version)
+        user_login_bonus = await self.data.item.get_all_login_bonus(user_id, self.version)
         # ignore the loginBonus request if its disabled in config
         if user_login_bonus is None or not self.game_cfg.mods.use_login_bonus:
             return {"userId": user_id, "length": 0, "userLoginBonusList": []}
@@ -552,8 +552,8 @@ class ChuniBase:
             "userLoginBonusList": user_login_list,
         }
 
-    def handle_get_user_map_api_request(self, data: Dict) -> Dict:
-        user_map_list = self.data.item.get_maps(data["userId"])
+    async def handle_get_user_map_api_request(self, data: Dict) -> Dict:
+        user_map_list = await self.data.item.get_maps(data["userId"])
         if user_map_list is None:
             return {}
 
@@ -570,8 +570,8 @@ class ChuniBase:
             "userMapList": map_list,
         }
 
-    def handle_get_user_music_api_request(self, data: Dict) -> Dict:
-        music_detail = self.data.score.get_scores(data["userId"])
+    async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
+        music_detail = await self.data.score.get_scores(data["userId"])
         if music_detail is None:
             return {
                 "userId": data["userId"],
@@ -629,8 +629,8 @@ class ChuniBase:
             "userMusicList": song_list,  # 240
         }
 
-    def handle_get_user_option_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_option(data["userId"])
+    async def handle_get_user_option_api_request(self, data: Dict) -> Dict:
+        p = await self.data.profile.get_profile_option(data["userId"])
 
         option = p._asdict()
         option.pop("id")
@@ -638,8 +638,8 @@ class ChuniBase:
 
         return {"userId": data["userId"], "userGameOption": option}
 
-    def handle_get_user_option_ex_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_option_ex(data["userId"])
+    async def handle_get_user_option_ex_api_request(self, data: Dict) -> Dict:
+        p = await self.data.profile.get_profile_option_ex(data["userId"])
 
         option = p._asdict()
         option.pop("id")
@@ -650,11 +650,11 @@ class ChuniBase:
     def read_wtf8(self, src):
         return bytes([ord(c) for c in src]).decode("utf-8")
 
-    def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_preview(data["userId"], self.version)
+    async def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
+        profile = await self.data.profile.get_profile_preview(data["userId"], self.version)
         if profile is None:
             return None
-        profile_character = self.data.item.get_character(
+        profile_character = await self.data.item.get_character(
             data["userId"], profile["characterId"]
         )
 
@@ -692,8 +692,8 @@ class ChuniBase:
             "userNameEx": profile["userName"],
         }
 
-    def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
-        recent_rating_list = self.data.profile.get_profile_recent_rating(data["userId"])
+    async def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
+        recent_rating_list = await self.data.profile.get_profile_recent_rating(data["userId"])
         if recent_rating_list is None:
             return {
                 "userId": data["userId"],
@@ -707,7 +707,7 @@ class ChuniBase:
             "userRecentRatingList": recent_rating_list["recentRating"],
         }
 
-    def handle_get_user_region_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_region_api_request(self, data: Dict) -> Dict:
         # TODO: Region
         return {
             "userId": data["userId"],
@@ -715,22 +715,22 @@ class ChuniBase:
             "userRegionList": [],
         }
 
-    def handle_get_user_team_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_team_api_request(self, data: Dict) -> Dict:
         # Default values
         team_id = 65535
         team_name = self.game_cfg.team.team_name
         team_rank = 0
 
         # Get user profile
-        profile = self.data.profile.get_profile_data(data["userId"], self.version)
+        profile = await self.data.profile.get_profile_data(data["userId"], self.version)
         if profile and profile["teamId"]:
             # Get team by id
-            team = self.data.profile.get_team_by_id(profile["teamId"])
+            team = await self.data.profile.get_team_by_id(profile["teamId"])
 
             if team:
                 team_id = team["id"]
                 team_name = team["teamName"]
-                team_rank = self.data.profile.get_team_rank(team["id"])
+                team_rank = await self.data.profile.get_team_rank(team["id"])
 
         # Don't return anything if no team name has been defined for defaults and there is no team set for the player
         if not profile["teamId"] and team_name == "":
@@ -750,7 +750,7 @@ class ChuniBase:
             },
         }
     
-    def handle_get_team_course_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_team_course_setting_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
@@ -758,7 +758,7 @@ class ChuniBase:
             "teamCourseSettingList": [],
         }
 
-    def handle_get_team_course_setting_api_request_proto(self, data: Dict) -> Dict:
+    async def handle_get_team_course_setting_api_request_proto(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 1,
@@ -782,7 +782,7 @@ class ChuniBase:
             ],
         }
 
-    def handle_get_team_course_rule_api_request(self, data: Dict) -> Dict:
+    async def handle_get_team_course_rule_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
@@ -790,7 +790,7 @@ class ChuniBase:
             "teamCourseRuleList": []
         }
 
-    def handle_get_team_course_rule_api_request_proto(self, data: Dict) -> Dict:
+    async def handle_get_team_course_rule_api_request_proto(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 1,
@@ -807,7 +807,7 @@ class ChuniBase:
             ],
         }
 
-    def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
         upsert = data["upsertUserAll"]
         user_id = data["userId"]
 
@@ -819,58 +819,58 @@ class ChuniBase:
             except Exception:
                 pass
 
-            self.data.profile.put_profile_data(
+            await self.data.profile.put_profile_data(
                 user_id, self.version, upsert["userData"][0]
             )
 
         if "userDataEx" in upsert:
-            self.data.profile.put_profile_data_ex(
+            await self.data.profile.put_profile_data_ex(
                 user_id, self.version, upsert["userDataEx"][0]
             )
 
         if "userGameOption" in upsert:
-            self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
+            await self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
 
         if "userGameOptionEx" in upsert:
-            self.data.profile.put_profile_option_ex(
+            await self.data.profile.put_profile_option_ex(
                 user_id, upsert["userGameOptionEx"][0]
             )
         if "userRecentRatingList" in upsert:
-            self.data.profile.put_profile_recent_rating(
+            await self.data.profile.put_profile_recent_rating(
                 user_id, upsert["userRecentRatingList"]
             )
 
         if "userCharacterList" in upsert:
             for character in upsert["userCharacterList"]:
-                self.data.item.put_character(user_id, character)
+                await self.data.item.put_character(user_id, character)
 
         if "userMapList" in upsert:
             for map in upsert["userMapList"]:
-                self.data.item.put_map(user_id, map)
+                await self.data.item.put_map(user_id, map)
 
         if "userCourseList" in upsert:
             for course in upsert["userCourseList"]:
-                self.data.score.put_course(user_id, course)
+                await self.data.score.put_course(user_id, course)
 
         if "userDuelList" in upsert:
             for duel in upsert["userDuelList"]:
-                self.data.item.put_duel(user_id, duel)
+                await self.data.item.put_duel(user_id, duel)
 
         if "userItemList" in upsert:
             for item in upsert["userItemList"]:
-                self.data.item.put_item(user_id, item)
+                await self.data.item.put_item(user_id, item)
 
         if "userActivityList" in upsert:
             for activity in upsert["userActivityList"]:
-                self.data.profile.put_profile_activity(user_id, activity)
+                await self.data.profile.put_profile_activity(user_id, activity)
 
         if "userChargeList" in upsert:
             for charge in upsert["userChargeList"]:
-                self.data.profile.put_profile_charge(user_id, charge)
+                await self.data.profile.put_profile_charge(user_id, charge)
 
         if "userMusicDetailList" in upsert:
             for song in upsert["userMusicDetailList"]:
-                self.data.score.put_score(user_id, song)
+                await self.data.score.put_score(user_id, song)
 
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
@@ -881,7 +881,7 @@ class ChuniBase:
                   playlog["playedUserName2"] = self.read_wtf8(playlog["playedUserName2"])
                 if playlog["playedUserName3"] is not None:
                   playlog["playedUserName3"] = self.read_wtf8(playlog["playedUserName3"])
-                self.data.score.put_playlog(user_id, playlog, self.version)
+                await self.data.score.put_playlog(user_id, playlog, self.version)
 
         if "userTeamPoint" in upsert:
             team_points = upsert["userTeamPoint"]
@@ -889,7 +889,7 @@ class ChuniBase:
                 for tp in team_points:
                     if tp["teamId"] != '65535':
                         # Fetch the current team data
-                        current_team = self.data.profile.get_team_by_id(tp["teamId"])
+                        current_team = await self.data.profile.get_team_by_id(tp["teamId"])
 
                         # Calculate the new teamPoint
                         new_team_point = int(tp["teamPoint"]) + current_team["teamPoint"]
@@ -900,24 +900,24 @@ class ChuniBase:
                         }
 
                         # Update the team data
-                        self.data.profile.update_team(tp["teamId"], team_data)
+                        await self.data.profile.update_team(tp["teamId"], team_data)
             except:
                 pass # Probably a better way to catch if the team is not set yet (new profiles), but let's just pass
         if "userMapAreaList" in upsert:
             for map_area in upsert["userMapAreaList"]:
-                self.data.item.put_map_area(user_id, map_area)
+                await self.data.item.put_map_area(user_id, map_area)
 
         if "userOverPowerList" in upsert:
             for overpower in upsert["userOverPowerList"]:
-                self.data.profile.put_profile_overpower(user_id, overpower)
+                await self.data.profile.put_profile_overpower(user_id, overpower)
 
         if "userEmoneyList" in upsert:
             for emoney in upsert["userEmoneyList"]:
-                self.data.profile.put_profile_emoney(user_id, emoney)
+                await self.data.profile.put_profile_emoney(user_id, emoney)
 
         if "userLoginBonusList" in upsert:
             for login in upsert["userLoginBonusList"]:
-                self.data.item.put_login_bonus(
+                await self.data.item.put_login_bonus(
                     user_id, self.version, login["presetId"], isWatched=True
                 )
         
@@ -925,30 +925,41 @@ class ChuniBase:
             for rp in upsert["userRecentPlayerList"]:
                 pass
 
+        for rating_type in {"userRatingBaseList", "userRatingBaseHotList", "userRatingBaseNextList"}:
+            if rating_type not in upsert:
+                continue
+            
+            await self.data.profile.put_profile_rating(
+                user_id,
+                self.version,
+                rating_type,
+                upsert[rating_type],
+            )
+
         return {"returnCode": "1"}
 
-    def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
         # add tickets after they got bought, this makes sure the tickets are
         # still valid after an unsuccessful logout
-        self.data.profile.put_profile_charge(data["userId"], data["userCharge"])
+        await self.data.profile.put_profile_charge(data["userId"], data["userCharge"])
         return {"returnCode": "1"}
 
-    def handle_upsert_client_bookkeeping_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_bookkeeping_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_develop_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_develop_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_error_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_error_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_setting_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_testmode_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_testmode_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_get_user_net_battle_data_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_net_battle_data_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "userNetBattleData": {"recentNBSelectMusicList": []},

titles/chuni/crystal.py

@@ -13,7 +13,7 @@ class ChuniCrystal(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_CRYSTAL
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.40.00"
         return ret

titles/chuni/crystalplus.py

@@ -13,7 +13,7 @@ class ChuniCrystalPlus(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.45.00"
         return ret

titles/chuni/frontend.py

@@ -0,0 +1,83 @@
+from typing import List
+from starlette.routing import Route
+from starlette.requests import Request
+from starlette.responses import Response, RedirectResponse
+from os import path
+import yaml
+import jinja2
+
+from core.frontend import FE_Base, UserSession
+from core.config import CoreConfig
+from .database import ChuniData
+from .config import ChuniConfig
+from .const import ChuniConstants
+
+
+class ChuniFrontend(FE_Base):
+    def __init__(
+        self, cfg: CoreConfig, environment: jinja2.Environment, cfg_dir: str
+    ) -> None:
+        super().__init__(cfg, environment)
+        self.data = ChuniData(cfg)
+        self.game_cfg = ChuniConfig()
+        if path.exists(f"{cfg_dir}/{ChuniConstants.CONFIG_NAME}"):
+            self.game_cfg.update(
+                yaml.safe_load(open(f"{cfg_dir}/{ChuniConstants.CONFIG_NAME}"))
+            )
+        self.nav_name = "Chunithm"
+
+    def get_routes(self) -> List[Route]:
+        return [
+            Route("/", self.render_GET, methods=['GET']),
+            Route("/update.name", self.update_name, methods=['POST']),
+        ]
+
+    async def render_GET(self, request: Request) -> bytes:
+        template = self.environment.get_template(
+            "titles/chuni/templates/chuni_index.jinja"
+        )
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh:
+            usr_sesh = UserSession()
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | {self.nav_name}",
+            game_list=self.environment.globals["game_list"],
+            sesh=vars(usr_sesh)
+        ), media_type="text/html; charset=utf-8")
+
+    async def update_name(self, request: Request) -> bytes:
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh:
+            return RedirectResponse("/gate/", 303)
+        
+        new_name: str = request.query_params.get('new_name', '')
+        new_name_full = ""
+        
+        if not new_name:
+            return RedirectResponse("/gate/?e=4", 303)
+        
+        if len(new_name) > 8:
+            return RedirectResponse("/gate/?e=8", 303)
+        
+        for x in new_name: # FIXME: This will let some invalid characters through atm
+            o = ord(x)
+            try:
+                if o == 0x20:
+                    new_name_full += chr(0x3000)
+                elif o < 0x7F and o > 0x20:
+                    new_name_full += chr(o + 0xFEE0)
+                elif o <= 0x7F:
+                    self.logger.warn(f"Invalid ascii character {o:02X}")
+                    return RedirectResponse("/gate/?e=4", 303)
+                else:
+                    new_name_full += x
+            
+            except Exception as e:
+                self.logger.error(f"Something went wrong parsing character {o:04X} - {e}")
+                return RedirectResponse("/gate/?e=4", 303)
+        
+        if not await self.data.profile.update_name(usr_sesh, new_name_full):
+            return RedirectResponse("/gate/?e=999", 303)
+
+        return RedirectResponse("/gate/?s=1", 303)

titles/chuni/index.py

@@ -1,4 +1,6 @@
-from twisted.web.http import Request
+from starlette.requests import Request
+from starlette.routing import Route
+from starlette.responses import Response
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 import zlib
@@ -33,7 +35,6 @@ from .newplus import ChuniNewPlus
 from .sun import ChuniSun
 from .sunplus import ChuniSunPlus
 
-
 class ChuniServlet(BaseServlet):
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
         super().__init__(core_cfg, cfg_dir)
@@ -124,15 +125,6 @@ class ChuniServlet(BaseServlet):
                     f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()}"
                 )
 
-    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
-        return (
-            [], 
-            [
-               ("render_POST", "/{game}/{version}/ChuniServlet/{endpoint}", {}),
-               ("render_POST", "/{game}/{version}/ChuniServlet/MatchingServer/{endpoint}", {})
-            ]
-        )
-
     @classmethod
     def is_game_enabled(
         cls, game_code: str, core_cfg: CoreConfig, cfg_dir: str
@@ -150,19 +142,25 @@ class ChuniServlet(BaseServlet):
 
     def get_allnet_info(self, game_code: str, game_ver: int, keychip: str) -> Tuple[str, str]:
         if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
-            return (f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", self.core_cfg.title.hostname)
+            return (f"http://{self.core_cfg.server.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", self.core_cfg.server.hostname)
 
-        return (f"http://{self.core_cfg.title.hostname}/{game_code}/{game_ver}/", self.core_cfg.title.hostname)
+        return (f"http://{self.core_cfg.server.hostname}/{game_code}/{game_ver}/", self.core_cfg.server.hostname)
 
-    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
-        endpoint = matchers['endpoint']
-        version = int(matchers['version'])
-        game_code = matchers['game']
+    def get_routes(self) -> List[Route]:
+        return [
+            Route("/{game:str}/{version:int}/ChuniServlet/{endpoint:str}", self.render_POST, methods=['POST']),
+            Route("/{game:str}/{version:int}/ChuniServlet/MatchingServer/{endpoint:str}", self.render_POST, methods=['POST']),
+        ]
+
+    async def render_POST(self, request: Request) -> bytes:
+        endpoint: str = request.path_params.get('endpoint')
+        version: int = request.path_params.get('version')
+        game_code: str = request.path_params.get('game')
 
         if endpoint.lower() == "ping":
-            return zlib.compress(b'{"returnCode": "1"}')
+            return Response(zlib.compress(b'{"returnCode": "1"}'))
 
-        req_raw = request.content.getvalue()
+        req_raw = await request.body()
 
         encrtped = False
         internal_ver = 0
@@ -201,7 +199,7 @@ class ChuniServlet(BaseServlet):
               internal_ver = ChuniConstants.VER_CHUNITHM_SUN_PLUS
         elif game_code == "SDGS": # Int
           if version < 110: # SUPERSTAR
-              internal_ver = ChuniConstants.PARADISE
+              internal_ver = ChuniConstants.VER_CHUNITHM_PARADISE # FIXME: Not sure what was intended to go here? was just "PARADISE"
           elif version >= 110 and version < 115: # NEW
               internal_ver = ChuniConstants.VER_CHUNITHM_NEW
           elif version >= 115 and version < 120: # NEW PLUS!!
@@ -216,20 +214,20 @@ class ChuniServlet(BaseServlet):
             # doing encrypted. The likelyhood of false positives is low but
             # technically not 0
             if internal_ver < ChuniConstants.VER_CHUNITHM_NEW:
-                endpoint = request.getHeader("User-Agent").split("#")[0]
+                endpoint = request.headers.get("User-Agent").split("#")[0]
 
             else:
                 if internal_ver not in self.hash_table:
                     self.logger.error(
                         f"v{version} does not support encryption or no keys entered"
                     )
-                    return zlib.compress(b'{"stat": "0"}')
+                    return Response(zlib.compress(b'{"stat": "0"}'))
 
                 elif endpoint.lower() not in self.hash_table[internal_ver]:
                     self.logger.error(
                         f"No hash found for v{version} endpoint {endpoint}"
                     )
-                    return zlib.compress(b'{"stat": "0"}')
+                    return Response(zlib.compress(b'{"stat": "0"}'))
 
                 endpoint = self.hash_table[internal_ver][endpoint.lower()]
 
@@ -246,7 +244,7 @@ class ChuniServlet(BaseServlet):
                 self.logger.error(
                     f"Failed to decrypt v{version} request to {endpoint} -> {e}"
                 )
-                return zlib.compress(b'{"stat": "0"}')
+                return Response(zlib.compress(b'{"stat": "0"}'))
 
             encrtped = True
 
@@ -258,7 +256,7 @@ class ChuniServlet(BaseServlet):
             self.logger.error(
                 f"Unencrypted v{version} {endpoint} request, but config is set to encrypted only: {req_raw}"
             )
-            return zlib.compress(b'{"stat": "0"}')
+            return Response(zlib.compress(b'{"stat": "0"}'))
 
         try:
             unzip = zlib.decompress(req_raw)
@@ -267,7 +265,7 @@ class ChuniServlet(BaseServlet):
             self.logger.error(
                 f"Failed to decompress v{version} {endpoint} request -> {e}"
             )
-            return b""
+            return Response(zlib.compress(b'{"stat": "0"}'))
 
         req_data = json.loads(unzip)
 
@@ -285,11 +283,11 @@ class ChuniServlet(BaseServlet):
         else:
             try:
                 handler = getattr(handler_cls, func_to_find)
-                resp = handler(req_data)
+                resp = await handler(req_data)
 
             except Exception as e:
                 self.logger.error(f"Error handling v{version} method {endpoint} - {e}")
-                return zlib.compress(b'{"stat": "0"}')
+                return Response(zlib.compress(b'{"stat": "0"}'))
 
         if resp == None:
             resp = {"returnCode": 1}
@@ -299,7 +297,7 @@ class ChuniServlet(BaseServlet):
         zipped = zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))
 
         if not encrtped:
-            return zipped
+            return Response(zipped)
 
         padded = pad(zipped, 16)
 
@@ -309,4 +307,4 @@ class ChuniServlet(BaseServlet):
             bytes.fromhex(self.game_cfg.crypto.keys[internal_ver][1]),
         )
 
-        return crypt.encrypt(padded)
+        return Response(crypt.encrypt(padded))

titles/chuni/new.py

@@ -33,7 +33,7 @@ class ChuniNew(ChuniBase):
         if self.version == ChuniConstants.VER_CHUNITHM_SUN_PLUS:
             return "215"
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         # use UTC time and convert it to JST time by adding +9
         # matching therefore starts one hour before and lasts for 8 hours
         match_start = datetime.strftime(
@@ -82,27 +82,27 @@ class ChuniNew(ChuniBase):
                 "matchErrorLimit": self.game_cfg.matching.match_error_limit,
                 "romVersion": self.game_cfg.version.version(self.version)["rom"],
                 "dataVersion": self.game_cfg.version.version(self.version)["data"],
-                "matchingUri": f"http://{self.core_cfg.title.hostname}:{t_port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
-                "matchingUriX": f"http://{self.core_cfg.title.hostname}:{t_port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
+                "matchingUri": f"http://{self.core_cfg.server.hostname}:{t_port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
+                "matchingUriX": f"http://{self.core_cfg.server.hostname}:{t_port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
                 # might be really important for online battle to connect the cabs via UDP port 50201
-                "udpHolePunchUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
-                "reflectorUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
+                "udpHolePunchUri": f"http://{self.core_cfg.server.hostname}:{self.core_cfg.server.port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
+                "reflectorUri": f"http://{self.core_cfg.server.hostname}:{self.core_cfg.server.port}/SDHD/{self._interal_ver_to_intver()}/ChuniServlet/",
             },
             "isDumpUpload": False,
             "isAou": False,
         }
 
-    def handle_remove_token_api_request(self, data: Dict) -> Dict:
+    async def handle_remove_token_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_delete_token_api_request(self, data: Dict) -> Dict:
+    async def handle_delete_token_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_create_token_api_request(self, data: Dict) -> Dict:
+    async def handle_create_token_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_get_user_map_area_api_request(self, data: Dict) -> Dict:
-        user_map_areas = self.data.item.get_map_areas(data["userId"])
+    async def handle_get_user_map_area_api_request(self, data: Dict) -> Dict:
+        user_map_areas = await self.data.item.get_map_areas(data["userId"])
 
         map_areas = []
         for map_area in user_map_areas:
@@ -113,14 +113,14 @@ class ChuniNew(ChuniBase):
 
         return {"userId": data["userId"], "userMapAreaList": map_areas}
 
-    def handle_get_user_symbol_chat_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_symbol_chat_setting_api_request(self, data: Dict) -> Dict:
         return {"userId": data["userId"], "symbolCharInfoList": []}
 
-    def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_preview(data["userId"], self.version)
+    async def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
+        profile = await self.data.profile.get_profile_preview(data["userId"], self.version)
         if profile is None:
             return None
-        profile_character = self.data.item.get_character(
+        profile_character = await self.data.item.get_character(
             data["userId"], profile["characterId"]
         )
 
@@ -164,8 +164,8 @@ class ChuniNew(ChuniBase):
         }
         return data1
 
-    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_data(data["userId"], self.version)
+    async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        p = await self.data.profile.get_profile_data(data["userId"], self.version)
         if p is None:
             return {}
 
@@ -177,17 +177,17 @@ class ChuniNew(ChuniBase):
             "isLogin": False,
         }
 
-    def handle_printer_login_api_request(self, data: Dict) -> Dict:
+    async def handle_printer_login_api_request(self, data: Dict) -> Dict:
         return {"returnCode": 1}
 
-    def handle_printer_logout_api_request(self, data: Dict) -> Dict:
+    async def handle_printer_logout_api_request(self, data: Dict) -> Dict:
         return {"returnCode": 1}
 
-    def handle_get_game_gacha_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_gacha_api_request(self, data: Dict) -> Dict:
         """
         returns all current active banners (gachas)
         """
-        game_gachas = self.data.static.get_gachas(self.version)
+        game_gachas = await self.data.static.get_gachas(self.version)
 
         # clean the database rows
         game_gacha_list = []
@@ -213,11 +213,11 @@ class ChuniNew(ChuniBase):
             "registIdList": [],
         }
 
-    def handle_get_game_gacha_card_by_id_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_gacha_card_by_id_api_request(self, data: Dict) -> Dict:
         """
         returns all valid cards for a given gachaId
         """
-        game_gacha_cards = self.data.static.get_gacha_cards(data["gachaId"])
+        game_gacha_cards = await self.data.static.get_gacha_cards(data["gachaId"])
 
         game_gacha_card_list = []
         for gacha_card in game_gacha_cards:
@@ -237,8 +237,8 @@ class ChuniNew(ChuniBase):
             "ssrBookCalcList": [],
         }
 
-    def handle_cm_get_user_data_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_data(data["userId"], self.version)
+    async def handle_cm_get_user_data_api_request(self, data: Dict) -> Dict:
+        p = await self.data.profile.get_profile_data(data["userId"], self.version)
         if p is None:
             return {}
 
@@ -262,8 +262,8 @@ class ChuniNew(ChuniBase):
             ],
         }
 
-    def handle_get_user_gacha_api_request(self, data: Dict) -> Dict:
-        user_gachas = self.data.item.get_user_gachas(data["userId"])
+    async def handle_get_user_gacha_api_request(self, data: Dict) -> Dict:
+        user_gachas = await self.data.item.get_user_gachas(data["userId"])
         if user_gachas is None:
             return {"userId": data["userId"], "length": 0, "userGachaList": []}
 
@@ -281,8 +281,8 @@ class ChuniNew(ChuniBase):
             "userGachaList": user_gacha_list,
         }
 
-    def handle_get_user_printed_card_api_request(self, data: Dict) -> Dict:
-        user_print_list = self.data.item.get_user_print_states(
+    async def handle_get_user_printed_card_api_request(self, data: Dict) -> Dict:
+        user_print_list = await self.data.item.get_user_print_states(
             data["userId"], has_completed=True
         )
         if user_print_list is None:
@@ -316,10 +316,10 @@ class ChuniNew(ChuniBase):
             "userPrintedCardList": print_list,
         }
 
-    def handle_get_user_card_print_error_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_card_print_error_api_request(self, data: Dict) -> Dict:
         user_id = data["userId"]
 
-        user_print_states = self.data.item.get_user_print_states(
+        user_print_states = await self.data.item.get_user_print_states(
             user_id, has_completed=False
         )
 
@@ -338,13 +338,13 @@ class ChuniNew(ChuniBase):
             "userCardPrintStateList": card_print_state_list,
         }
 
-    def handle_cm_get_user_character_api_request(self, data: Dict) -> Dict:
-        return super().handle_get_user_character_api_request(data)
+    async def handle_cm_get_user_character_api_request(self, data: Dict) -> Dict:
+        return await super().handle_get_user_character_api_request(data)
 
-    def handle_cm_get_user_item_api_request(self, data: Dict) -> Dict:
-        return super().handle_get_user_item_api_request(data)
+    async def handle_cm_get_user_item_api_request(self, data: Dict) -> Dict:
+        return await super().handle_get_user_item_api_request(data)
 
-    def handle_roll_gacha_api_request(self, data: Dict) -> Dict:
+    async def handle_roll_gacha_api_request(self, data: Dict) -> Dict:
         """
         Handle a gacha roll API request, with:
         gachaId: the gachaId where the cards should be pulled from
@@ -362,14 +362,14 @@ class ChuniNew(ChuniBase):
         # characterId should be returned
         if chara_id != -1:
             # get the
-            card = self.data.static.get_gacha_card_by_character(gacha_id, chara_id)
+            card = await self.data.static.get_gacha_card_by_character(gacha_id, chara_id)
 
             tmp = card._asdict()
             tmp.pop("id")
 
             rolled_cards.append(tmp)
         else:
-            gacha_cards = self.data.static.get_gacha_cards(gacha_id)
+            gacha_cards = await self.data.static.get_gacha_cards(gacha_id)
 
             # get the card id for each roll
             for _ in range(num_rolls):
@@ -386,7 +386,7 @@ class ChuniNew(ChuniBase):
 
         return {"length": len(rolled_cards), "gameGachaCardList": rolled_cards}
 
-    def handle_cm_upsert_user_gacha_api_request(self, data: Dict) -> Dict:
+    async def handle_cm_upsert_user_gacha_api_request(self, data: Dict) -> Dict:
         upsert = data["cmUpsertUserGacha"]
         user_id = data["userId"]
         place_id = data["placeId"]
@@ -396,7 +396,7 @@ class ChuniNew(ChuniBase):
         user_data.pop("rankUpChallengeResults")
         user_data.pop("userEmoney")
 
-        self.data.profile.put_profile_data(user_id, self.version, user_data)
+        await self.data.profile.put_profile_data(user_id, self.version, user_data)
 
         # save the user gacha
         user_gacha = upsert["userGacha"]
@@ -404,16 +404,16 @@ class ChuniNew(ChuniBase):
         user_gacha.pop("gachaId")
         user_gacha.pop("dailyGachaDate")
 
-        self.data.item.put_user_gacha(user_id, gacha_id, user_gacha)
+        await self.data.item.put_user_gacha(user_id, gacha_id, user_gacha)
 
         # save all user items
         if "userItemList" in upsert:
             for item in upsert["userItemList"]:
-                self.data.item.put_item(user_id, item)
+                await self.data.item.put_item(user_id, item)
 
         # add every gamegachaCard to database
         for card in upsert["gameGachaCardList"]:
-            self.data.item.put_user_print_state(
+            await self.data.item.put_user_print_state(
                 user_id,
                 hasCompleted=False,
                 placeId=place_id,
@@ -423,7 +423,7 @@ class ChuniNew(ChuniBase):
 
         # retrieve every game gacha card which has been added in order to get
         # the orderId for the next request
-        user_print_states = self.data.item.get_user_print_states_by_gacha(
+        user_print_states = await self.data.item.get_user_print_states_by_gacha(
             user_id, gacha_id, has_completed=False
         )
         card_print_state_list = []
@@ -441,7 +441,7 @@ class ChuniNew(ChuniBase):
             "userCardPrintStateList": card_print_state_list,
         }
 
-    def handle_cm_upsert_user_printlog_api_request(self, data: Dict) -> Dict:
+    async def handle_cm_upsert_user_printlog_api_request(self, data: Dict) -> Dict:
         return {
             "returnCode": 1,
             "orderId": 0,
@@ -449,7 +449,7 @@ class ChuniNew(ChuniBase):
             "apiName": "CMUpsertUserPrintlogApi",
         }
 
-    def handle_cm_upsert_user_print_api_request(self, data: Dict) -> Dict:
+    async def handle_cm_upsert_user_print_api_request(self, data: Dict) -> Dict:
         user_print_detail = data["userPrintDetail"]
         user_id = data["userId"]
 
@@ -465,7 +465,7 @@ class ChuniNew(ChuniBase):
         )
 
         # add the entry to the user print table with the random serialId
-        self.data.item.put_user_print_detail(user_id, serial_id, user_print_detail)
+        await self.data.item.put_user_print_detail(user_id, serial_id, user_print_detail)
 
         return {
             "returnCode": 1,
@@ -474,7 +474,7 @@ class ChuniNew(ChuniBase):
             "apiName": "CMUpsertUserPrintApi",
         }
 
-    def handle_cm_upsert_user_print_subtract_api_request(self, data: Dict) -> Dict:
+    async def handle_cm_upsert_user_print_subtract_api_request(self, data: Dict) -> Dict:
         upsert = data["userCardPrintState"]
         user_id = data["userId"]
         place_id = data["placeId"]
@@ -482,37 +482,37 @@ class ChuniNew(ChuniBase):
         # save all user items
         if "userItemList" in data:
             for item in data["userItemList"]:
-                self.data.item.put_item(user_id, item)
+                await self.data.item.put_item(user_id, item)
 
         # set the card print state to success and use the orderId as the key
-        self.data.item.put_user_print_state(
+        await self.data.item.put_user_print_state(
             user_id, id=upsert["orderId"], hasCompleted=True
         )
 
         return {"returnCode": "1", "apiName": "CMUpsertUserPrintSubtractApi"}
 
-    def handle_cm_upsert_user_print_cancel_api_request(self, data: Dict) -> Dict:
+    async def handle_cm_upsert_user_print_cancel_api_request(self, data: Dict) -> Dict:
         order_ids = data["orderIdList"]
         user_id = data["userId"]
 
         # set the card print state to success and use the orderId as the key
         for order_id in order_ids:
-            self.data.item.put_user_print_state(user_id, id=order_id, hasCompleted=True)
+            await self.data.item.put_user_print_state(user_id, id=order_id, hasCompleted=True)
 
         return {"returnCode": "1", "apiName": "CMUpsertUserPrintCancelApi"}
 
-    def handle_ping_request(self, data: Dict) -> Dict:
+    async def handle_ping_request(self, data: Dict) -> Dict:
         # matchmaking ping request
         return {"returnCode": "1"}
 
-    def handle_begin_matching_api_request(self, data: Dict) -> Dict:
+    async def handle_begin_matching_api_request(self, data: Dict) -> Dict:
         room_id = 1
         # check if there is a free matching room
-        matching_room = self.data.item.get_oldest_free_matching(self.version)
+        matching_room = await self.data.item.get_oldest_free_matching(self.version)
 
         if matching_room is None:
             # grab the latest roomId and add 1 for the new room
-            newest_matching = self.data.item.get_newest_matching(self.version)
+            newest_matching = await self.data.item.get_newest_matching(self.version)
             if newest_matching is not None:
                 room_id = newest_matching["roomId"] + 1
 
@@ -522,12 +522,12 @@ class ChuniNew(ChuniBase):
 
             # create the new room with room_id and the current user id (host)
             # user id is required for the countdown later on
-            self.data.item.put_matching(
+            await self.data.item.put_matching(
                 self.version, room_id, [new_member], user_id=new_member["userId"]
             )
 
             # get the newly created matching room
-            matching_room = self.data.item.get_matching(self.version, room_id)
+            matching_room = await self.data.item.get_matching(self.version, room_id)
         else:
             # a room already exists, so just add the new member to it
             matching_member_list = matching_room["matchingMemberInfoList"]
@@ -537,7 +537,7 @@ class ChuniNew(ChuniBase):
             matching_member_list.append(new_member)
 
             # add the updated room to the database, make sure to set isFull correctly!
-            self.data.item.put_matching(
+            await self.data.item.put_matching(
                 self.version,
                 matching_room["roomId"],
                 matching_member_list,
@@ -554,8 +554,8 @@ class ChuniNew(ChuniBase):
 
         return {"roomId": 1, "matchingWaitState": matching_wait}
 
-    def handle_end_matching_api_request(self, data: Dict) -> Dict:
-        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+    async def handle_end_matching_api_request(self, data: Dict) -> Dict:
+        matching_room = await self.data.item.get_matching(self.version, data["roomId"])
         members = matching_room["matchingMemberInfoList"]
 
         # only set the host user to role 1 every other to 0?
@@ -564,7 +564,7 @@ class ChuniNew(ChuniBase):
             for m in members
         ]
 
-        self.data.item.put_matching(
+        await self.data.item.put_matching(
             self.version,
             matching_room["roomId"],
             members,
@@ -579,13 +579,13 @@ class ChuniNew(ChuniBase):
             # no idea, maybe to differentiate between CPUs and real players?
             "matchingMemberRoleList": role_list,
             # TCP/UDP connection?
-            "reflectorUri": f"{self.core_cfg.title.hostname}",
+            "reflectorUri": f"{self.core_cfg.server.hostname}",
         }
 
-    def handle_remove_matching_member_api_request(self, data: Dict) -> Dict:
+    async def handle_remove_matching_member_api_request(self, data: Dict) -> Dict:
         # get all matching rooms, because Chuni only returns the userId
         # not the actual roomId
-        matching_rooms = self.data.item.get_all_matchings(self.version)
+        matching_rooms = await self.data.item.get_all_matchings(self.version)
         if matching_rooms is None:
             return {"returnCode": "1"}
 
@@ -599,10 +599,10 @@ class ChuniNew(ChuniBase):
 
             # if the last user got removed, delete the matching room
             if len(new_members) <= 0:
-                self.data.item.delete_matching(self.version, room["roomId"])
+                await self.data.item.delete_matching(self.version, room["roomId"])
             else:
                 # remove the user from the room
-                self.data.item.put_matching(
+                await self.data.item.put_matching(
                     self.version,
                     room["roomId"],
                     new_members,
@@ -612,10 +612,10 @@ class ChuniNew(ChuniBase):
 
         return {"returnCode": "1"}
 
-    def handle_get_matching_state_api_request(self, data: Dict) -> Dict:
+    async def handle_get_matching_state_api_request(self, data: Dict) -> Dict:
         polling_interval = 1
         # get the current active room
-        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        matching_room = await self.data.item.get_matching(self.version, data["roomId"])
         members = matching_room["matchingMemberInfoList"]
         rest_sec = matching_room["restMSec"]
 
@@ -638,7 +638,7 @@ class ChuniNew(ChuniBase):
                 current_member["userName"] = self.read_wtf8(current_member["userName"])
                 members[i] = current_member
 
-        self.data.item.put_matching(
+        await self.data.item.put_matching(
             self.version,
             data["roomId"],
             members,

titles/chuni/newplus.py

@@ -11,8 +11,8 @@ class ChuniNewPlus(ChuniNew):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_NEW_PLUS
 
-    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
-        user_data = super().handle_cm_get_user_preview_api_request(data)
+    async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = await super().handle_cm_get_user_preview_api_request(data)
 
         # hardcode lastDataVersion for CardMaker 1.35 A028
         user_data["lastDataVersion"] = "2.05.00"

titles/chuni/paradise.py

@@ -13,7 +13,7 @@ class ChuniParadise(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_PARADISE
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.50.00"
         return ret

titles/chuni/plus.py

@@ -11,7 +11,7 @@ class ChuniPlus(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_PLUS
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.05.00"
         return ret

titles/chuni/read.py

@@ -28,7 +28,7 @@ class ChuniReader(BaseReader):
             self.logger.error(f"Invalid chunithm version {version}")
             exit(1)
 
-    def read(self) -> None:
+    async def read(self) -> None:
         data_dirs = []
         if self.bin_dir is not None:
             data_dirs += self.get_data_directories(self.bin_dir)
@@ -38,13 +38,13 @@ class ChuniReader(BaseReader):
 
         for dir in data_dirs:
             self.logger.info(f"Read from {dir}")
-            self.read_events(f"{dir}/event")
-            self.read_music(f"{dir}/music")
-            self.read_charges(f"{dir}/chargeItem")
-            self.read_avatar(f"{dir}/avatarAccessory")
-            self.read_login_bonus(f"{dir}/")
+            await self.read_events(f"{dir}/event")
+            await self.read_music(f"{dir}/music")
+            await self.read_charges(f"{dir}/chargeItem")
+            await self.read_avatar(f"{dir}/avatarAccessory")
+            await self.read_login_bonus(f"{dir}/")
 
-    def read_login_bonus(self, root_dir: str) -> None:
+    async def read_login_bonus(self, root_dir: str) -> None:
         for root, dirs, files in walk(f"{root_dir}loginBonusPreset"):
             for dir in dirs:
                 if path.exists(f"{root}/{dir}/LoginBonusPreset.xml"):
@@ -60,7 +60,7 @@ class ChuniReader(BaseReader):
                         True if xml_root.find("disableFlag").text == "false" else False
                     )
 
-                    result = self.data.static.put_login_bonus_preset(
+                    result = await self.data.static.put_login_bonus_preset(
                         self.version, id, name, is_enabled
                     )
 
@@ -98,7 +98,7 @@ class ChuniReader(BaseReader):
                                     bonus_root.find("loginBonusCategoryType").text
                                 )
 
-                                result = self.data.static.put_login_bonus(
+                                result = await self.data.static.put_login_bonus(
                                     self.version,
                                     id,
                                     bonus_id,
@@ -117,7 +117,7 @@ class ChuniReader(BaseReader):
                                         f"Failed to insert login bonus {bonus_id}"
                                     )
 
-    def read_events(self, evt_dir: str) -> None:
+    async def read_events(self, evt_dir: str) -> None:
         for root, dirs, files in walk(evt_dir):
             for dir in dirs:
                 if path.exists(f"{root}/{dir}/Event.xml"):
@@ -132,7 +132,7 @@ class ChuniReader(BaseReader):
                     for substances in xml_root.findall("substances"):
                         event_type = substances.find("type").text
 
-                    result = self.data.static.put_event(
+                    result = await self.data.static.put_event(
                         self.version, id, event_type, name
                     )
                     if result is not None:
@@ -140,7 +140,7 @@ class ChuniReader(BaseReader):
                     else:
                         self.logger.warning(f"Failed to insert event {id}")
 
-    def read_music(self, music_dir: str) -> None:
+    async def read_music(self, music_dir: str) -> None:
         for root, dirs, files in walk(music_dir):
             for dir in dirs:
                 if path.exists(f"{root}/{dir}/Music.xml"):
@@ -185,7 +185,7 @@ class ChuniReader(BaseReader):
                                     )
                                     we_chara = None
 
-                                result = self.data.static.put_music(
+                                result = await self.data.static.put_music(
                                     self.version,
                                     song_id,
                                     chart_id,
@@ -206,7 +206,7 @@ class ChuniReader(BaseReader):
                                         f"Failed to insert music {song_id} chart {chart_id}"
                                     )
 
-    def read_charges(self, charge_dir: str) -> None:
+    async def read_charges(self, charge_dir: str) -> None:
         for root, dirs, files in walk(charge_dir):
             for dir in dirs:
                 if path.exists(f"{root}/{dir}/ChargeItem.xml"):
@@ -222,7 +222,7 @@ class ChuniReader(BaseReader):
                     consumeType = xml_root.find("consumeType").text
                     sellingAppeal = bool(xml_root.find("sellingAppeal").text)
 
-                    result = self.data.static.put_charge(
+                    result = await self.data.static.put_charge(
                         self.version,
                         id,
                         name,
@@ -236,7 +236,7 @@ class ChuniReader(BaseReader):
                     else:
                         self.logger.warning(f"Failed to insert charge {id}")
 
-    def read_avatar(self, avatar_dir: str) -> None:
+    async def read_avatar(self, avatar_dir: str) -> None:
         for root, dirs, files in walk(avatar_dir):
             for dir in dirs:
                 if path.exists(f"{root}/{dir}/AvatarAccessory.xml"):
@@ -254,7 +254,7 @@ class ChuniReader(BaseReader):
                     for texture in xml_root.findall("texture"):
                         texturePath = texture.find("path").text
 
-                    result = self.data.static.put_avatar(
+                    result = await self.data.static.put_avatar(
                         self.version, id, name, category, iconPath, texturePath
                     )
 

titles/chuni/schema/item.py

@@ -245,7 +245,7 @@ matching = Table(
 
 
 class ChuniItemData(BaseData):
-    def get_oldest_free_matching(self, version: int) -> Optional[Row]:
+    async def get_oldest_free_matching(self, version: int) -> Optional[Row]:
         sql = matching.select(
             and_(
                 matching.c.version == version,
@@ -253,46 +253,46 @@ class ChuniItemData(BaseData):
             )
         ).order_by(matching.c.roomId.asc())
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_newest_matching(self, version: int) -> Optional[Row]:
+    async def get_newest_matching(self, version: int) -> Optional[Row]:
         sql = matching.select(
             and_(
                 matching.c.version == version
             )
         ).order_by(matching.c.roomId.desc())
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_all_matchings(self, version: int) -> Optional[List[Row]]:
+    async def get_all_matchings(self, version: int) -> Optional[List[Row]]:
         sql = matching.select(
             and_(
                 matching.c.version == version
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_matching(self, version: int, room_id: int) -> Optional[Row]:
+    async def get_matching(self, version: int, room_id: int) -> Optional[Row]:
         sql = matching.select(
             and_(matching.c.version == version, matching.c.roomId == room_id)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_matching(
+    async def put_matching(
         self,
         version: int,
         room_id: int,
@@ -314,22 +314,22 @@ class ChuniItemData(BaseData):
             restMSec=rest_sec, matchingMemberInfoList=matching_member_info_list
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def delete_matching(self, version: int, room_id: int):
+    async def delete_matching(self, version: int, room_id: int):
         sql = delete(matching).where(
             and_(matching.c.roomId == room_id, matching.c.version == version)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_all_favorites(
+    async def get_all_favorites(
         self, user_id: int, version: int, fav_kind: int = 1
     ) -> Optional[List[Row]]:
         sql = favorite.select(
@@ -340,12 +340,12 @@ class ChuniItemData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_login_bonus(
+    async def put_login_bonus(
         self, user_id: int, version: int, preset_id: int, **login_bonus_data
     ) -> Optional[int]:
         sql = insert(login_bonus).values(
@@ -354,12 +354,12 @@ class ChuniItemData(BaseData):
 
         conflict = sql.on_duplicate_key_update(presetId=preset_id, **login_bonus_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_all_login_bonus(
+    async def get_all_login_bonus(
         self, user_id: int, version: int, is_finished: bool = False
     ) -> Optional[List[Row]]:
         sql = login_bonus.select(
@@ -370,12 +370,12 @@ class ChuniItemData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_login_bonus(
+    async def get_login_bonus(
         self, user_id: int, version: int, preset_id: int
     ) -> Optional[Row]:
         sql = login_bonus.select(
@@ -386,12 +386,12 @@ class ChuniItemData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_character(self, user_id: int, character_data: Dict) -> Optional[int]:
+    async def put_character(self, user_id: int, character_data: Dict) -> Optional[int]:
         character_data["user"] = user_id
 
         character_data = self.fix_bools(character_data)
@@ -399,30 +399,30 @@ class ChuniItemData(BaseData):
         sql = insert(character).values(**character_data)
         conflict = sql.on_duplicate_key_update(**character_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_character(self, user_id: int, character_id: int) -> Optional[Dict]:
+    async def get_character(self, user_id: int, character_id: int) -> Optional[Dict]:
         sql = select(character).where(
             and_(character.c.user == user_id, character.c.characterId == character_id)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_characters(self, user_id: int) -> Optional[List[Row]]:
+    async def get_characters(self, user_id: int) -> Optional[List[Row]]:
         sql = select(character).where(character.c.user == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_item(self, user_id: int, item_data: Dict) -> Optional[int]:
+    async def put_item(self, user_id: int, item_data: Dict) -> Optional[int]:
         item_data["user"] = user_id
 
         item_data = self.fix_bools(item_data)
@@ -430,12 +430,12 @@ class ChuniItemData(BaseData):
         sql = insert(item).values(**item_data)
         conflict = sql.on_duplicate_key_update(**item_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_items(self, user_id: int, kind: int = None) -> Optional[List[Row]]:
+    async def get_items(self, user_id: int, kind: int = None) -> Optional[List[Row]]:
         if kind is None:
             sql = select(item).where(item.c.user == user_id)
         else:
@@ -443,12 +443,12 @@ class ChuniItemData(BaseData):
                 and_(item.c.user == user_id, item.c.itemKind == kind)
             )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_duel(self, user_id: int, duel_data: Dict) -> Optional[int]:
+    async def put_duel(self, user_id: int, duel_data: Dict) -> Optional[int]:
         duel_data["user"] = user_id
 
         duel_data = self.fix_bools(duel_data)
@@ -456,20 +456,20 @@ class ChuniItemData(BaseData):
         sql = insert(duel).values(**duel_data)
         conflict = sql.on_duplicate_key_update(**duel_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_duels(self, user_id: int) -> Optional[List[Row]]:
+    async def get_duels(self, user_id: int) -> Optional[List[Row]]:
         sql = select(duel).where(duel.c.user == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_map(self, user_id: int, map_data: Dict) -> Optional[int]:
+    async def put_map(self, user_id: int, map_data: Dict) -> Optional[int]:
         map_data["user"] = user_id
 
         map_data = self.fix_bools(map_data)
@@ -477,20 +477,20 @@ class ChuniItemData(BaseData):
         sql = insert(map).values(**map_data)
         conflict = sql.on_duplicate_key_update(**map_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_maps(self, user_id: int) -> Optional[List[Row]]:
+    async def get_maps(self, user_id: int) -> Optional[List[Row]]:
         sql = select(map).where(map.c.user == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_map_area(self, user_id: int, map_area_data: Dict) -> Optional[int]:
+    async def put_map_area(self, user_id: int, map_area_data: Dict) -> Optional[int]:
         map_area_data["user"] = user_id
 
         map_area_data = self.fix_bools(map_area_data)
@@ -498,28 +498,28 @@ class ChuniItemData(BaseData):
         sql = insert(map_area).values(**map_area_data)
         conflict = sql.on_duplicate_key_update(**map_area_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_map_areas(self, user_id: int) -> Optional[List[Row]]:
+    async def get_map_areas(self, user_id: int) -> Optional[List[Row]]:
         sql = select(map_area).where(map_area.c.user == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_user_gachas(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_user_gachas(self, aime_id: int) -> Optional[List[Row]]:
         sql = gacha.select(gacha.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_user_gacha(
+    async def put_user_gacha(
         self, aime_id: int, gacha_id: int, gacha_data: Dict
     ) -> Optional[int]:
         sql = insert(gacha).values(user=aime_id, gachaId=gacha_id, **gacha_data)
@@ -527,14 +527,14 @@ class ChuniItemData(BaseData):
         conflict = sql.on_duplicate_key_update(
             user=aime_id, gachaId=gacha_id, **gacha_data
         )
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
-    def get_user_print_states(
+    async def get_user_print_states(
         self, aime_id: int, has_completed: bool = False
     ) -> Optional[List[Row]]:
         sql = print_state.select(
@@ -544,12 +544,12 @@ class ChuniItemData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_user_print_states_by_gacha(
+    async def get_user_print_states_by_gacha(
         self, aime_id: int, gacha_id: int, has_completed: bool = False
     ) -> Optional[List[Row]]:
         sql = print_state.select(
@@ -560,16 +560,16 @@ class ChuniItemData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_user_print_state(self, aime_id: int, **print_data) -> Optional[int]:
+    async def put_user_print_state(self, aime_id: int, **print_data) -> Optional[int]:
         sql = insert(print_state).values(user=aime_id, **print_data)
 
         conflict = sql.on_duplicate_key_update(user=aime_id, **print_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(
@@ -578,7 +578,7 @@ class ChuniItemData(BaseData):
             return None
         return result.lastrowid
 
-    def put_user_print_detail(
+    async def put_user_print_detail(
         self, aime_id: int, serial_id: str, user_print_data: Dict
     ) -> Optional[int]:
         sql = insert(print_detail).values(
@@ -586,7 +586,7 @@ class ChuniItemData(BaseData):
         )
 
         conflict = sql.on_duplicate_key_update(user=aime_id, **user_print_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(

titles/chuni/schema/profile.py

@@ -1,10 +1,9 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
-from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger
-from sqlalchemy.engine.base import Connection
+from sqlalchemy import Table, Column, UniqueConstraint, and_
+from sqlalchemy.types import Integer, String, Boolean, JSON, BigInteger
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.engine import Row
-from sqlalchemy.sql import func, select
+from sqlalchemy.sql import select, delete
 from sqlalchemy.dialects.mysql import insert
 
 from core.data.schema import BaseData, metadata
@@ -393,9 +392,40 @@ team = Table(
     mysql_charset="utf8mb4",
 )
 
+rating = Table(
+    "chuni_profile_rating",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("type", String(255), nullable=False),
+    Column("index", Integer, nullable=False),
+    Column("musicId", Integer),
+    Column("difficultId", Integer),
+    Column("romVersionCode", Integer),
+    Column("score", Integer),
+    UniqueConstraint("user", "version", "type", "index", name="chuni_profile_rating_best_uk"),
+    mysql_charset="utf8mb4",
+)
+
 
 class ChuniProfileData(BaseData):
-    def put_profile_data(
+    async def update_name(self, user_id: int, new_name: str) -> bool:
+        sql = profile.update(profile.c.user == user_id).values(
+            userName=new_name
+        )
+        result = await self.execute(sql)
+
+        if result is None:
+            self.logger.warning(f"Failed to set user {user_id} name to {new_name}")
+            return False
+        return True
+
+    async def put_profile_data(
         self, aime_id: int, version: int, profile_data: Dict
     ) -> Optional[int]:
         profile_data["user"] = aime_id
@@ -407,26 +437,26 @@ class ChuniProfileData(BaseData):
 
         sql = insert(profile).values(**profile_data)
         conflict = sql.on_duplicate_key_update(**profile_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(f"put_profile_data: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
-    def get_profile_preview(self, aime_id: int, version: int) -> Optional[Row]:
+    async def get_profile_preview(self, aime_id: int, version: int) -> Optional[Row]:
         sql = (
             select([profile, option])
             .join(option, profile.c.user == option.c.user)
             .filter(and_(profile.c.user == aime_id, profile.c.version <= version))
         ).order_by(profile.c.version.desc())
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_profile_data(self, aime_id: int, version: int) -> Optional[Row]:
+    async def get_profile_data(self, aime_id: int, version: int) -> Optional[Row]:
         sql = select(profile).where(
             and_(
                 profile.c.user == aime_id,
@@ -434,12 +464,12 @@ class ChuniProfileData(BaseData):
             )
         ).order_by(profile.c.version.desc())
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_profile_data_ex(
+    async def put_profile_data_ex(
         self, aime_id: int, version: int, profile_ex_data: Dict
     ) -> Optional[int]:
         profile_ex_data["user"] = aime_id
@@ -449,7 +479,7 @@ class ChuniProfileData(BaseData):
 
         sql = insert(profile_ex).values(**profile_ex_data)
         conflict = sql.on_duplicate_key_update(**profile_ex_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(
@@ -458,7 +488,7 @@ class ChuniProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def get_profile_data_ex(self, aime_id: int, version: int) -> Optional[Row]:
+    async def get_profile_data_ex(self, aime_id: int, version: int) -> Optional[Row]:
         sql = select(profile_ex).where(
             and_(
                 profile_ex.c.user == aime_id,
@@ -466,17 +496,17 @@ class ChuniProfileData(BaseData):
             )
         ).order_by(profile_ex.c.version.desc())
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_profile_option(self, aime_id: int, option_data: Dict) -> Optional[int]:
+    async def put_profile_option(self, aime_id: int, option_data: Dict) -> Optional[int]:
         option_data["user"] = aime_id
 
         sql = insert(option).values(**option_data)
         conflict = sql.on_duplicate_key_update(**option_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(
@@ -485,22 +515,22 @@ class ChuniProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def get_profile_option(self, aime_id: int) -> Optional[Row]:
+    async def get_profile_option(self, aime_id: int) -> Optional[Row]:
         sql = select(option).where(option.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_profile_option_ex(
+    async def put_profile_option_ex(
         self, aime_id: int, option_ex_data: Dict
     ) -> Optional[int]:
         option_ex_data["user"] = aime_id
 
         sql = insert(option_ex).values(**option_ex_data)
         conflict = sql.on_duplicate_key_update(**option_ex_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(
@@ -509,15 +539,15 @@ class ChuniProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def get_profile_option_ex(self, aime_id: int) -> Optional[Row]:
+    async def get_profile_option_ex(self, aime_id: int) -> Optional[Row]:
         sql = select(option_ex).where(option_ex.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_profile_recent_rating(
+    async def put_profile_recent_rating(
         self, aime_id: int, recent_rating_data: List[Dict]
     ) -> Optional[int]:
         sql = insert(recent_rating).values(
@@ -525,7 +555,7 @@ class ChuniProfileData(BaseData):
         )
         conflict = sql.on_duplicate_key_update(recentRating=recent_rating_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             self.logger.warning(
                 f"put_profile_recent_rating: Failed to update! aime_id: {aime_id}"
@@ -533,15 +563,15 @@ class ChuniProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def get_profile_recent_rating(self, aime_id: int) -> Optional[Row]:
+    async def get_profile_recent_rating(self, aime_id: int) -> Optional[Row]:
         sql = select(recent_rating).where(recent_rating.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_profile_activity(self, aime_id: int, activity_data: Dict) -> Optional[int]:
+    async def put_profile_activity(self, aime_id: int, activity_data: Dict) -> Optional[int]:
         # The game just uses "id" but we need to distinguish that from the db column "id"
         activity_data["user"] = aime_id
         activity_data["activityId"] = activity_data["id"]
@@ -549,7 +579,7 @@ class ChuniProfileData(BaseData):
 
         sql = insert(activity).values(**activity_data)
         conflict = sql.on_duplicate_key_update(**activity_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(
@@ -558,24 +588,24 @@ class ChuniProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def get_profile_activity(self, aime_id: int, kind: int) -> Optional[List[Row]]:
+    async def get_profile_activity(self, aime_id: int, kind: int) -> Optional[List[Row]]:
         sql = (
             select(activity)
             .where(and_(activity.c.user == aime_id, activity.c.kind == kind))
             .order_by(activity.c.sortNumber.desc())  # to get the last played track
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_profile_charge(self, aime_id: int, charge_data: Dict) -> Optional[int]:
+    async def put_profile_charge(self, aime_id: int, charge_data: Dict) -> Optional[int]:
         charge_data["user"] = aime_id
 
         sql = insert(charge).values(**charge_data)
         conflict = sql.on_duplicate_key_update(**charge_data)
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warning(
@@ -584,40 +614,40 @@ class ChuniProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def get_profile_charge(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_profile_charge(self, aime_id: int) -> Optional[List[Row]]:
         sql = select(charge).where(charge.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def add_profile_region(self, aime_id: int, region_id: int) -> Optional[int]:
+    async def add_profile_region(self, aime_id: int, region_id: int) -> Optional[int]:
         pass
 
-    def get_profile_regions(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_profile_regions(self, aime_id: int) -> Optional[List[Row]]:
         pass
 
-    def put_profile_emoney(self, aime_id: int, emoney_data: Dict) -> Optional[int]:
+    async def put_profile_emoney(self, aime_id: int, emoney_data: Dict) -> Optional[int]:
         emoney_data["user"] = aime_id
 
         sql = insert(emoney).values(**emoney_data)
         conflict = sql.on_duplicate_key_update(**emoney_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_profile_emoney(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_profile_emoney(self, aime_id: int) -> Optional[List[Row]]:
         sql = select(emoney).where(emoney.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_profile_overpower(
+    async def put_profile_overpower(
         self, aime_id: int, overpower_data: Dict
     ) -> Optional[int]:
         overpower_data["user"] = aime_id
@@ -625,31 +655,31 @@ class ChuniProfileData(BaseData):
         sql = insert(overpower).values(**overpower_data)
         conflict = sql.on_duplicate_key_update(**overpower_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_profile_overpower(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_profile_overpower(self, aime_id: int) -> Optional[List[Row]]:
         sql = select(overpower).where(overpower.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_team_by_id(self, team_id: int) -> Optional[Row]:
+    async def get_team_by_id(self, team_id: int) -> Optional[Row]:
         sql = select(team).where(team.c.id == team_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             return None
         return result.fetchone()
 
-    def get_team_rank(self, team_id: int) -> int:
+    async def get_team_rank(self, team_id: int) -> int:
         # Normal ranking system, likely the one used in the real servers
         # Query all teams sorted by 'teamPoint'
-        result = self.execute(
+        result = await self.execute(
             select(team.c.id).order_by(team.c.teamPoint.desc())
         )
 
@@ -666,13 +696,13 @@ class ChuniProfileData(BaseData):
     # RIP scaled team ranking. Gone, but forgotten
     # def get_team_rank_scaled(self, team_id: int) -> int:
 
-    def update_team(self, team_id: int, team_data: Dict) -> bool:
+    async def update_team(self, team_id: int, team_data: Dict) -> bool:
         team_data["id"] = team_id
 
         sql = insert(team).values(**team_data)
         conflict = sql.on_duplicate_key_update(**team_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
 
         if result is None:
             self.logger.warn(
@@ -680,16 +710,16 @@ class ChuniProfileData(BaseData):
             )
             return False
         return True
-    def get_rival(self, rival_id: int) -> Optional[Row]:
+    async def get_rival(self, rival_id: int) -> Optional[Row]:
         sql = select(profile).where(profile.c.user == rival_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             return None
         return result.fetchone()
-    def get_overview(self) -> Dict:
+    async def get_overview(self) -> Dict:
         # Fetch and add up all the playcounts
-        playcount_sql = self.execute(select(profile.c.playCount))
+        playcount_sql = await self.execute(select(profile.c.playCount))
 
         if playcount_sql is None:
             self.logger.warn(
@@ -697,9 +727,33 @@ class ChuniProfileData(BaseData):
             )
             return 0
 
-        total_play_count = 0;
+        total_play_count = 0
         for row in playcount_sql:
             total_play_count += row[0]
         return {
             "total_play_count": total_play_count
         }
+    
+    async def put_profile_rating(
+        self,
+        aime_id: int,
+        version: int,
+        rating_type: str,
+        rating_data: List[Dict],
+    ):
+        inserted_values = [
+            {"user": aime_id, "version": version, "type": rating_type, "index": i, **x}
+            for (i, x) in enumerate(rating_data)
+        ]
+        sql = insert(rating).values(inserted_values)
+        update_dict = {x.name: x for x in sql.inserted if x.name != "id"}
+        sql = sql.on_duplicate_key_update(**update_dict)
+        result = await self.execute(sql)
+
+        if result is None:
+            self.logger.warn(
+                f"put_profile_rating: Could not insert {rating_type}, aime_id: {aime_id}",
+            )
+            return
+        
+        return result.lastrowid

titles/chuni/schema/score.py

@@ -142,55 +142,55 @@ playlog = Table(
 
 
 class ChuniScoreData(BaseData):
-    def get_courses(self, aime_id: int) -> Optional[Row]:
+    async def get_courses(self, aime_id: int) -> Optional[Row]:
         sql = select(course).where(course.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_course(self, aime_id: int, course_data: Dict) -> Optional[int]:
+    async def put_course(self, aime_id: int, course_data: Dict) -> Optional[int]:
         course_data["user"] = aime_id
         course_data = self.fix_bools(course_data)
 
         sql = insert(course).values(**course_data)
         conflict = sql.on_duplicate_key_update(**course_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_scores(self, aime_id: int) -> Optional[Row]:
+    async def get_scores(self, aime_id: int) -> Optional[Row]:
         sql = select(best_score).where(best_score.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_score(self, aime_id: int, score_data: Dict) -> Optional[int]:
+    async def put_score(self, aime_id: int, score_data: Dict) -> Optional[int]:
         score_data["user"] = aime_id
         score_data = self.fix_bools(score_data)
 
         sql = insert(best_score).values(**score_data)
         conflict = sql.on_duplicate_key_update(**score_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_playlogs(self, aime_id: int) -> Optional[Row]:
+    async def get_playlogs(self, aime_id: int) -> Optional[Row]:
         sql = select(playlog).where(playlog.c.user == aime_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_playlog(self, aime_id: int, playlog_data: Dict, version: int) -> Optional[int]:
+    async def put_playlog(self, aime_id: int, playlog_data: Dict, version: int) -> Optional[int]:
         # Calculate the ROM version that should be inserted into the DB, based on the version of the ggame being inserted
         # We only need from Version 10 (Plost) and back, as newer versions include romVersion in their upsert
         # This matters both for gameRankings, as well as a future DB update to keep version data separate
@@ -216,12 +216,12 @@ class ChuniScoreData(BaseData):
         sql = insert(playlog).values(**playlog_data)
         conflict = sql.on_duplicate_key_update(**playlog_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_rankings(self, version: int) -> Optional[List[Dict]]:
+    async def get_rankings(self, version: int) -> Optional[List[Dict]]:
         # Calculates the ROM version that should be fetched for rankings, based on the game version being retrieved
         # This prevents tracks that are not accessible in your version from counting towards the 10 results
         romVer = {
@@ -241,7 +241,7 @@ class ChuniScoreData(BaseData):
             0: "1.00%"
         }
         sql = select([playlog.c.musicId.label('id'), func.count(playlog.c.musicId).label('point')]).where((playlog.c.level != 4) & (playlog.c.romVersion.like(romVer.get(version, "%")))).group_by(playlog.c.musicId).order_by(func.count(playlog.c.musicId).desc()).limit(10)
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             return None
@@ -249,10 +249,10 @@ class ChuniScoreData(BaseData):
         rows = result.fetchall()
         return [dict(row) for row in rows]
 
-    def get_rival_music(self, rival_id: int) -> Optional[List[Dict]]:
+    async def get_rival_music(self, rival_id: int) -> Optional[List[Dict]]:
         sql = select(best_score).where(best_score.c.user == rival_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()

titles/chuni/schema/static.py

@@ -175,7 +175,7 @@ login_bonus = Table(
 
 
 class ChuniStaticData(BaseData):
-    def put_login_bonus(
+    async def put_login_bonus(
         self,
         version: int,
         preset_id: int,
@@ -207,12 +207,12 @@ class ChuniStaticData(BaseData):
             loginBonusCategoryType=login_bonus_category_type,
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_login_bonus(
+    async def get_login_bonus(
         self,
         version: int,
         preset_id: int,
@@ -224,12 +224,12 @@ class ChuniStaticData(BaseData):
             )
         ).order_by(login_bonus.c.needLoginDayCount.desc())
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_login_bonus_by_required_days(
+    async def get_login_bonus_by_required_days(
         self, version: int, preset_id: int, need_login_day_count: int
     ) -> Optional[Row]:
         sql = login_bonus.select(
@@ -240,12 +240,12 @@ class ChuniStaticData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_login_bonus_preset(
+    async def put_login_bonus_preset(
         self, version: int, preset_id: int, preset_name: str, is_enabled: bool
     ) -> Optional[int]:
         sql = insert(login_bonus_preset).values(
@@ -259,12 +259,12 @@ class ChuniStaticData(BaseData):
             presetName=preset_name, isEnabled=is_enabled
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_login_bonus_presets(
+    async def get_login_bonus_presets(
         self, version: int, is_enabled: bool = True
     ) -> Optional[List[Row]]:
         sql = login_bonus_preset.select(
@@ -274,12 +274,12 @@ class ChuniStaticData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_event(
+    async def put_event(
         self, version: int, event_id: int, type: int, name: str
     ) -> Optional[int]:
         sql = insert(events).values(
@@ -288,19 +288,19 @@ class ChuniStaticData(BaseData):
 
         conflict = sql.on_duplicate_key_update(name=name)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def update_event(
+    async def update_event(
         self, version: int, event_id: int, enabled: bool
     ) -> Optional[bool]:
         sql = events.update(
             and_(events.c.version == version, events.c.eventId == event_id)
         ).values(enabled=enabled)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.warning(
                 f"update_event: failed to update event! version: {version}, event_id: {event_id}, enabled: {enabled}"
@@ -315,35 +315,35 @@ class ChuniStaticData(BaseData):
             return None
         return event["enabled"]
 
-    def get_event(self, version: int, event_id: int) -> Optional[Row]:
+    async def get_event(self, version: int, event_id: int) -> Optional[Row]:
         sql = select(events).where(
             and_(events.c.version == version, events.c.eventId == event_id)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_enabled_events(self, version: int) -> Optional[List[Row]]:
+    async def get_enabled_events(self, version: int) -> Optional[List[Row]]:
         sql = select(events).where(
             and_(events.c.version == version, events.c.enabled == True)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_events(self, version: int) -> Optional[List[Row]]:
+    async def get_events(self, version: int) -> Optional[List[Row]]:
         sql = select(events).where(events.c.version == version)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_music(
+    async def put_music(
         self,
         version: int,
         song_id: int,
@@ -376,12 +376,12 @@ class ChuniStaticData(BaseData):
             worldsEndTag=we_tag,
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def put_charge(
+    async def put_charge(
         self,
         version: int,
         charge_id: int,
@@ -406,38 +406,38 @@ class ChuniStaticData(BaseData):
             sellingAppeal=selling_appeal,
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_enabled_charges(self, version: int) -> Optional[List[Row]]:
+    async def get_enabled_charges(self, version: int) -> Optional[List[Row]]:
         sql = select(charge).where(
             and_(charge.c.version == version, charge.c.enabled == True)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_charges(self, version: int) -> Optional[List[Row]]:
+    async def get_charges(self, version: int) -> Optional[List[Row]]:
         sql = select(charge).where(charge.c.version == version)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_music(self, version: int) -> Optional[List[Row]]:
+    async def get_music(self, version: int) -> Optional[List[Row]]:
         sql = music.select(music.c.version <= version)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_music_chart(
+    async def get_music_chart(
         self, version: int, song_id: int, chart_id: int
     ) -> Optional[List[Row]]:
         sql = select(music).where(
@@ -448,21 +448,21 @@ class ChuniStaticData(BaseData):
             )
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_song(self, music_id: int) -> Optional[Row]:
+    async def get_song(self, music_id: int) -> Optional[Row]:
         sql = music.select(music.c.id == music_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
 
-    def put_avatar(
+    async def put_avatar(
         self,
         version: int,
         avatarAccessoryId: int,
@@ -487,12 +487,12 @@ class ChuniStaticData(BaseData):
             texturePath=texturePath,
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def put_gacha(
+    async def put_gacha(
         self,
         version: int,
         gacha_id: int,
@@ -513,33 +513,33 @@ class ChuniStaticData(BaseData):
             **gacha_data,
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             self.logger.warning(f"Failed to insert gacha! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
-    def get_gachas(self, version: int) -> Optional[List[Dict]]:
+    async def get_gachas(self, version: int) -> Optional[List[Dict]]:
         sql = gachas.select(gachas.c.version <= version).order_by(
             gachas.c.gachaId.asc()
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_gacha(self, version: int, gacha_id: int) -> Optional[Dict]:
+    async def get_gacha(self, version: int, gacha_id: int) -> Optional[Dict]:
         sql = gachas.select(
             and_(gachas.c.version <= version, gachas.c.gachaId == gacha_id)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_gacha_card(
+    async def put_gacha_card(
         self, gacha_id: int, card_id: int, **gacha_card
     ) -> Optional[int]:
         sql = insert(gacha_cards).values(gachaId=gacha_id, cardId=card_id, **gacha_card)
@@ -548,21 +548,21 @@ class ChuniStaticData(BaseData):
             gachaId=gacha_id, cardId=card_id, **gacha_card
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             self.logger.warning(f"Failed to insert gacha card! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
-    def get_gacha_cards(self, gacha_id: int) -> Optional[List[Dict]]:
+    async def get_gacha_cards(self, gacha_id: int) -> Optional[List[Dict]]:
         sql = gacha_cards.select(gacha_cards.c.gachaId == gacha_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_gacha_card_by_character(
+    async def get_gacha_card_by_character(
         self, gacha_id: int, chara_id: int
     ) -> Optional[Dict]:
         sql_sub = (
@@ -574,26 +574,26 @@ class ChuniStaticData(BaseData):
             and_(gacha_cards.c.gachaId == gacha_id, gacha_cards.c.cardId == sql_sub)
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_card(self, version: int, card_id: int, **card_data) -> Optional[int]:
+    async def put_card(self, version: int, card_id: int, **card_data) -> Optional[int]:
         sql = insert(cards).values(version=version, cardId=card_id, **card_data)
 
         conflict = sql.on_duplicate_key_update(**card_data)
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             self.logger.warning(f"Failed to insert card! card_id {card_id}")
             return None
         return result.lastrowid
 
-    def get_card(self, version: int, card_id: int) -> Optional[Dict]:
+    async def get_card(self, version: int, card_id: int) -> Optional[Dict]:
         sql = cards.select(and_(cards.c.version <= version, cards.c.cardId == card_id))
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()

titles/chuni/star.py

@@ -11,7 +11,7 @@ class ChuniStar(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_STAR
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.20.00"
         return ret

titles/chuni/starplus.py

@@ -11,7 +11,7 @@ class ChuniStarPlus(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_STAR_PLUS
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.25.00"
         return ret

titles/chuni/sun.py

@@ -11,8 +11,8 @@ class ChuniSun(ChuniNewPlus):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_SUN
 
-    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
-        user_data = super().handle_cm_get_user_preview_api_request(data)
+    async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = await super().handle_cm_get_user_preview_api_request(data)
 
         # hardcode lastDataVersion for CardMaker 1.35 A032
         user_data["lastDataVersion"] = "2.10.00"

titles/chuni/sunplus.py

@@ -11,8 +11,8 @@ class ChuniSunPlus(ChuniSun):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_SUN_PLUS
 
-    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
-        user_data = super().handle_cm_get_user_preview_api_request(data)
+    async def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = await super().handle_cm_get_user_preview_api_request(data)
 
         # I don't know if lastDataVersion is going to matter, I don't think CardMaker 1.35 works this far up
         user_data["lastDataVersion"] = "2.15.00"

titles/chuni/templates/chuni_index.jinja

@@ -0,0 +1,43 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<h1>Chunithm</h1>
+{% if profile is defined and profile is not none and profile.id > 0 %}
+<script type="text/javascript">
+function toggle_new_name_form() {
+  let frm = document.getElementById("new_name_form");
+  let btn = document.getElementById("btn_toggle_form");
+
+  if (frm.style['display'] != "") {
+    frm.style['display'] = "";
+    frm.style['max-height'] = "";
+    btn.innerText = "Cancel";
+  } else {
+    frm.style['display'] = "none";
+    frm.style['max-height'] = "0px";
+    btn.innerText = "Edit";
+  }
+}
+</script>
+<h3>Profile for {{ profile.userName }}&nbsp;<button onclick="toggle_new_name_form()" class="btn btn-secondary" id="btn_toggle_form">Edit</button></h3>
+{% if error is defined %}
+{% include "core/templates/widgets/err_banner.jinja" %}
+{% endif %}
+{% if success is defined and success == 1 %}
+<div style="background-color: #00AA00; padding: 20px; margin-bottom: 10px; width: 15%;">
+Update successful
+</div>
+{% endif %}
+<form style="max-width: 33%; display: none; max-height: 0px;" action="/game/chuni/update.name" method="post" id="new_name_form">
+  <div class="mb-3">
+    <label for="new_name" class="form-label">New Trainer Name</label>
+    <input type="text" class="form-control" id="new_name" name="new_name" aria-describedby="new_name_help" maxlength="14">
+    <div id="new_name_help" class="form-text">Must be 14 characters or less</div>
+  </div>
+  <button type="submit" class="btn btn-primary">Submit</button>
+</form>
+{% elif sesh is defined and sesh is not none and sesh.user_id > 0 %}
+No profile information found for this account.
+{% else %}
+Login to view profile information.
+{% endif %}
+{% endblock content %}

titles/cm/__init__.py

@@ -6,7 +6,4 @@ from titles.cm.database import CardMakerData
 index = CardMakerServlet
 reader = CardMakerReader
 database = CardMakerData
-
 game_codes = [CardMakerConstants.GAME_CODE]
-
-current_schema_version = 1

titles/cm/base.py

@@ -29,11 +29,11 @@ class CardMakerBase:
     def _parse_int_ver(version: str) -> str:
         return version.replace(".", "")[:3]
 
-    def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
         if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
-            uri = f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}"
+            uri = f"http://{self.core_cfg.server.hostname}:{Utils.get_title_port(self.core_cfg)}"
         else:
-            uri = f"http://{self.core_cfg.title.hostname}"
+            uri = f"http://{self.core_cfg.server.hostname}"
 
         # grab the dict with all games version numbers from user config
         games_ver = self.game_cfg.version.version(self.version)
@@ -62,7 +62,7 @@ class CardMakerBase:
             ],
         }
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         # if reboot start/end time is not defined use the default behavior of being a few hours ago
         if self.core_cfg.title.reboot_start_time == "" or self.core_cfg.title.reboot_end_time == "":
             reboot_start = datetime.strftime(
@@ -110,11 +110,11 @@ class CardMakerBase:
             "isAou": False,
         }
 
-    def handle_get_client_bookkeeping_api_request(self, data: Dict) -> Dict:
+    async def handle_get_client_bookkeeping_api_request(self, data: Dict) -> Dict:
         return {"placeId": data["placeId"], "length": 0, "clientBookkeepingList": []}
 
-    def handle_upsert_client_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_setting_api_request(self, data: Dict) -> Dict:
         return {"returnCode": 1, "apiName": "UpsertClientSettingApi"}
 
-    def handle_upsert_client_bookkeeping_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_bookkeeping_api_request(self, data: Dict) -> Dict:
         return {"returnCode": 1, "apiName": "UpsertClientBookkeepingApi"}

titles/cm/cm135.py

@@ -12,7 +12,7 @@ class CardMaker135(CardMakerBase):
         super().__init__(core_cfg, game_cfg)
         self.version = CardMakerConstants.VER_CARD_MAKER_135
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
         return ret

titles/cm/index.py

@@ -5,10 +5,11 @@ import string
 import logging
 import coloredlogs
 import zlib
-
+from starlette.routing import Route
+from starlette.responses import Response
+from starlette.requests import Request
 from os import path
-from typing import Tuple, List, Dict
-from twisted.web.http import Request
+from typing import List
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
@@ -19,7 +20,6 @@ from .const import CardMakerConstants
 from .base import CardMakerBase
 from .cm135 import CardMaker135
 
-
 class CardMakerServlet(BaseServlet):
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
         super().__init__(core_cfg, cfg_dir)
@@ -72,16 +72,15 @@ class CardMakerServlet(BaseServlet):
 
         return True
     
-    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
-        return (
-            [], 
-            [("render_POST", "/SDED/{version}/{endpoint}", {})]
-        )
-
-    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
-        version = int(matchers['version'])
-        endpoint = matchers['endpoint']
-        req_raw = request.content.getvalue()
+    def get_routes(self) -> List[Route]:
+        return [
+            Route("/SDED/{version:int}/{endpoint:str}", self.render_POST, methods=['POST'])
+        ]
+    
+    async def render_POST(self, request: Request) -> bytes:
+        version: int = request.path_params.get('version')
+        endpoint: str = request.path_params.get('endpoint')
+        req_raw = await request.body()
         internal_ver = 0
         client_ip = Utils.get_ip_addr(request)
 
@@ -103,7 +102,7 @@ class CardMakerServlet(BaseServlet):
             self.logger.error(
                 f"Failed to decompress v{version} {endpoint} request -> {e}"
             )
-            return zlib.compress(b'{"stat": "0"}')
+            return Response(zlib.compress(b'{"stat": "0"}'))
 
         req_data = json.loads(unzip)
 
@@ -114,20 +113,20 @@ class CardMakerServlet(BaseServlet):
 
         if not hasattr(self.versions[internal_ver], func_to_find):
             self.logger.warning(f"Unhandled v{version} request {endpoint}")
-            return zlib.compress(b'{"returnCode": 1}')
+            return Response(zlib.compress(b'{"returnCode": 1}'))
 
         try:
             handler = getattr(self.versions[internal_ver], func_to_find)
-            resp = handler(req_data)
+            resp = await handler(req_data)
 
         except Exception as e:
             self.logger.error(f"Error handling v{version} method {endpoint} - {e}")
             raise
-            return zlib.compress(b'{"stat": "0"}')
+            return Response(zlib.compress(b'{"stat": "0"}'))
 
         if resp is None:
             resp = {"returnCode": 1}
 
         self.logger.debug(f"Response {resp}")
 
-        return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))
+        return Response(zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8")))

titles/cm/read.py

@@ -50,7 +50,7 @@ class CardMakerReader(BaseReader):
                 ):
                     return f"{root}/{dir}"
 
-    def read(self) -> None:
+    async def read(self) -> None:
         static_datas = {
             "static_gachas.csv": "read_ongeki_gacha_csv",
             "static_gacha_cards.csv": "read_ongeki_gacha_card_csv",
@@ -59,14 +59,14 @@ class CardMakerReader(BaseReader):
         if self.bin_dir is not None:
             data_dir = self._get_card_maker_directory(self.bin_dir)
 
-            self.read_chuni_card(f"{data_dir}/CHU/Data/A000/card")
-            self.read_chuni_gacha(f"{data_dir}/CHU/Data/A000/gacha")
+            await self.read_chuni_card(f"{data_dir}/CHU/Data/A000/card")
+            await self.read_chuni_gacha(f"{data_dir}/CHU/Data/A000/gacha")
 
-            self.read_mai2_card(f"{data_dir}/MAI/Data/A000/card")
+            await self.read_mai2_card(f"{data_dir}/MAI/Data/A000/card")
             for file, func in static_datas.items():
                 if os.path.exists(f"{self.bin_dir}/MU3/{file}"):
                     read_csv = getattr(CardMakerReader, func)
-                    read_csv(self, f"{self.bin_dir}/MU3/{file}")
+                    await read_csv(self, f"{self.bin_dir}/MU3/{file}")
                 else:
                     self.logger.warning(
                         f"Couldn't find {file} file in {self.bin_dir}, skipping"
@@ -78,12 +78,12 @@ class CardMakerReader(BaseReader):
             # ONGEKI (MU3) cnnot easily access the bin data(A000.pac)
             # so only opt_dir will work for now
             for dir in data_dirs:
-                self.read_chuni_card(f"{dir}/CHU/card")
-                self.read_chuni_gacha(f"{dir}/CHU/gacha")
-                self.read_mai2_card(f"{dir}/MAI/card")
-                self.read_ongeki_gacha(f"{dir}/MU3/gacha")
+                await self.read_chuni_card(f"{dir}/CHU/card")
+                await self.read_chuni_gacha(f"{dir}/CHU/gacha")
+                await self.read_mai2_card(f"{dir}/MAI/card")
+                await self.read_ongeki_gacha(f"{dir}/MU3/gacha")
 
-    def read_chuni_card(self, base_dir: str) -> None:
+    async def read_chuni_card(self, base_dir: str) -> None:
         self.logger.info(f"Reading cards from {base_dir}...")
 
         version_ids = {
@@ -114,7 +114,7 @@ class CardMakerReader(BaseReader):
                         chain = int(troot.find("chain").text)
                         skill_name = troot.find("skillName").text
 
-                        self.chuni_data.static.put_card(
+                        await self.chuni_data.static.put_card(
                             version,
                             card_id,
                             charaName=chara_name,
@@ -131,7 +131,7 @@ class CardMakerReader(BaseReader):
 
                         self.logger.info(f"Added chuni card {card_id}")
 
-    def read_chuni_gacha(self, base_dir: str) -> None:
+    async def read_chuni_gacha(self, base_dir: str) -> None:
         self.logger.info(f"Reading gachas from {base_dir}...")
 
         version_ids = {
@@ -158,7 +158,7 @@ class CardMakerReader(BaseReader):
                             True if troot.find("ceilingType").text == "1" else False
                         )
 
-                        self.chuni_data.static.put_gacha(
+                        await self.chuni_data.static.put_gacha(
                             version,
                             gacha_id,
                             name,
@@ -181,7 +181,7 @@ class CardMakerReader(BaseReader):
                                 True if gacha_card.find("pickup").text == "1" else False
                             )
 
-                            self.chuni_data.static.put_gacha_card(
+                            await self.chuni_data.static.put_gacha_card(
                                 gacha_id,
                                 card_id,
                                 weight=weight,
@@ -193,7 +193,7 @@ class CardMakerReader(BaseReader):
                                 f"Added chuni card {card_id} to gacha {gacha_id}"
                             )
 
-    def read_mai2_card(self, base_dir: str) -> None:
+    async def read_mai2_card(self, base_dir: str) -> None:
         self.logger.info(f"Reading cards from {base_dir}...")
 
         version_ids = {
@@ -206,6 +206,7 @@ class CardMakerReader(BaseReader):
             "1.25": Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS,
             "1.30": Mai2Constants.VER_MAIMAI_DX_FESTIVAL,
             "1.35": Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS,
+            "1.40": Mai2Constants.VER_MAIMAI_DX_BUDDIES,
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -225,24 +226,18 @@ class CardMakerReader(BaseReader):
                             True if troot.find("disable").text == "false" else False
                         )
 
-                        # check if a date is part of the name and disable the
-                        # card if it is
-                        enabled = (
-                            False if re.search(r"\d{2}/\d{2}/\d{2}", name) else enabled
-                        )
-
-                        self.mai2_data.static.put_card(
+                        await self.mai2_data.static.put_card(
                             version, card_id, name, enabled=enabled
                         )
                         self.logger.info(f"Added mai2 card {card_id}")
 
-    def read_ongeki_gacha_csv(self, file_path: str) -> None:
+    async def read_ongeki_gacha_csv(self, file_path: str) -> None:
         self.logger.info(f"Reading gachas from {file_path}...")
 
         with open(file_path, encoding="utf-8") as f:
             reader = csv.DictReader(f)
             for row in reader:
-                self.ongeki_data.static.put_gacha(
+                await self.ongeki_data.static.put_gacha(
                     row["version"],
                     row["gachaId"],
                     row["gachaName"],
@@ -254,13 +249,13 @@ class CardMakerReader(BaseReader):
 
                 self.logger.info(f"Added ongeki gacha {row['gachaId']}")
 
-    def read_ongeki_gacha_card_csv(self, file_path: str) -> None:
+    async def read_ongeki_gacha_card_csv(self, file_path: str) -> None:
         self.logger.info(f"Reading gacha cards from {file_path}...")
 
         with open(file_path, encoding="utf-8") as f:
             reader = csv.DictReader(f)
             for row in reader:
-                self.ongeki_data.static.put_gacha_card(
+                await self.ongeki_data.static.put_gacha_card(
                     row["gachaId"],
                     row["cardId"],
                     rarity=row["rarity"],
@@ -271,7 +266,7 @@ class CardMakerReader(BaseReader):
 
                 self.logger.info(f"Added ongeki card {row['cardId']} to gacha")
 
-    def read_ongeki_gacha(self, base_dir: str) -> None:
+    async def read_ongeki_gacha(self, base_dir: str) -> None:
         self.logger.info(f"Reading gachas from {base_dir}...")
 
         # assuming some GachaKinds based on the GachaType
@@ -294,7 +289,7 @@ class CardMakerReader(BaseReader):
 
                         # skip already existing gachas
                         if (
-                            self.ongeki_data.static.get_gacha(
+                            await self.ongeki_data.static.get_gacha(
                                 OngekiConstants.VER_ONGEKI_BRIGHT_MEMORY, gacha_id
                             )
                             is not None
@@ -320,7 +315,7 @@ class CardMakerReader(BaseReader):
                             is_ceiling = 1
                             max_select_point = 33
 
-                        self.ongeki_data.static.put_gacha(
+                        await self.ongeki_data.static.put_gacha(
                             version,
                             gacha_id,
                             name,

titles/cxb/__init__.py

@@ -7,4 +7,3 @@ index = CxbServlet
 database = CxbData
 reader = CxbReader
 game_codes = [CxbConstants.GAME_CODE]
-current_schema_version = 1

titles/cxb/base.py

@@ -28,14 +28,14 @@ class CxbBase:
         
         return []
 
-    def handle_action_rpreq_request(self, data: Dict) -> Dict:
+    async def handle_action_rpreq_request(self, data: Dict) -> Dict:
         return {}
 
-    def handle_action_hitreq_request(self, data: Dict) -> Dict:
+    async def handle_action_hitreq_request(self, data: Dict) -> Dict:
         return {"data": []}
 
-    def handle_auth_usercheck_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_index(
+    async def handle_auth_usercheck_request(self, data: Dict) -> Dict:
+        profile = await self.data.profile.get_profile_index(
             0, data["usercheck"]["authid"], self.version
         )
         if profile is not None:
@@ -45,12 +45,12 @@ class CxbBase:
         self.logger.info(f"No profile for aime id {data['usercheck']['authid']}")
         return {"exist": "false", "logout": "true"}
 
-    def handle_auth_entry_request(self, data: Dict) -> Dict:
+    async def handle_auth_entry_request(self, data: Dict) -> Dict:
         self.logger.info(f"New profile for {data['entry']['authid']}")
         return {"token": data["entry"]["authid"], "uid": data["entry"]["authid"]}
 
-    def handle_auth_login_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_index(
+    async def handle_auth_login_request(self, data: Dict) -> Dict:
+        profile = await self.data.profile.get_profile_index(
             0, data["login"]["authid"], self.version
         )
 
@@ -198,14 +198,14 @@ class CxbBase:
             ).decode("utf-8")
         )
 
-    def handle_action_loadrange_request(self, data: Dict) -> Dict:
+    async def handle_action_loadrange_request(self, data: Dict) -> Dict:
         range_start = data["loadrange"]["range"][0]
         range_end = data["loadrange"]["range"][1]
         uid = data["loadrange"]["uid"]
 
         self.logger.info(f"Load data for {uid}")
-        profile = self.data.profile.get_profile(uid, self.version)
-        songs = self.data.score.get_best_scores(uid)
+        profile = await self.data.profile.get_profile(uid, self.version)
+        songs = await self.data.score.get_best_scores(uid)
 
         data1 = []
         index = []
@@ -271,7 +271,7 @@ class CxbBase:
             thread_ScoreData = Thread(target=CxbBase.task_generateScoreData(song, index, data1))
             thread_ScoreData.start()
 
-        v_profile = self.data.profile.get_profile_index(0, uid, self.version)
+        v_profile = await self.data.profile.get_profile_index(0, uid, self.version)
         v_profile_data = v_profile["data"]
 
         for _, data in enumerate(profile):
@@ -282,7 +282,7 @@ class CxbBase:
 
         return {"index": index, "data": data1, "version": versionindex}
 
-    def handle_action_saveindex_request(self, data: Dict) -> Dict:
+    async def handle_action_saveindex_request(self, data: Dict) -> Dict:
         save_data = data["saveindex"]
 
         try:
@@ -300,11 +300,11 @@ class CxbBase:
 
             for value in data["saveindex"]["data"]:
                 if "playedUserId" in value[1]:
-                    self.data.profile.put_profile(
+                    await self.data.profile.put_profile(
                         data["saveindex"]["uid"], self.version, value[0], value[1]
                     )
                 if "mcode" not in value[1]:
-                    self.data.profile.put_profile(
+                    await self.data.profile.put_profile(
                         data["saveindex"]["uid"], self.version, value[0], value[1]
                     )
                 if "shopId" in value:
@@ -335,7 +335,7 @@ class CxbBase:
                             "index": value[0],
                         }
                     )
-                    self.data.score.put_best_score(
+                    await self.data.score.put_best_score(
                         data["saveindex"]["uid"],
                         song_json["mcode"],
                         self.version,
@@ -360,32 +360,32 @@ class CxbBase:
 
         for index, value in enumerate(data["saveindex"]["data"]):
             if int(data["saveindex"]["index"][index]) == 101:
-                self.data.profile.put_profile(
+                await self.data.profile.put_profile(
                     aimeId, self.version, data["saveindex"]["index"][index], value
                 )
             if (
                 int(data["saveindex"]["index"][index]) >= 700000
                 and int(data["saveindex"]["index"][index]) <= 701000
             ):
-                self.data.profile.put_profile(
+                await self.data.profile.put_profile(
                     aimeId, self.version, data["saveindex"]["index"][index], value
                 )
             if (
                 int(data["saveindex"]["index"][index]) >= 500
                 and int(data["saveindex"]["index"][index]) <= 510
             ):
-                self.data.profile.put_profile(
+                await self.data.profile.put_profile(
                     aimeId, self.version, data["saveindex"]["index"][index], value
                 )
             if "playedUserId" in value:
-                self.data.profile.put_profile(
+                await self.data.profile.put_profile(
                     aimeId,
                     self.version,
                     data["saveindex"]["index"][index],
                     json.loads(value),
                 )
             if "mcode" not in value and "normalCR" not in value:
-                self.data.profile.put_profile(
+                await self.data.profile.put_profile(
                     aimeId,
                     self.version,
                     data["saveindex"]["index"][index],
@@ -437,16 +437,16 @@ class CxbBase:
                 }
             )
 
-            self.data.score.put_best_score(
+            await self.data.score.put_best_score(
                 aimeId, data1["mcode"], self.version, indexSongList[i], songCode[0]
             )
             i += 1
         return {}
 
-    def handle_action_sprankreq_request(self, data: Dict) -> Dict:
+    async def handle_action_sprankreq_request(self, data: Dict) -> Dict:
         uid = data["sprankreq"]["uid"]
         self.logger.info(f"Get best rankings for {uid}")
-        p = self.data.score.get_best_rankings(uid)
+        p = await self.data.score.get_best_rankings(uid)
 
         rankList: List[Dict[str, Any]] = []
 
@@ -475,16 +475,16 @@ class CxbBase:
             "rankx": [1, 1, 1],
         }
 
-    def handle_action_getadv_request(self, data: Dict) -> Dict:
+    async def handle_action_getadv_request(self, data: Dict) -> Dict:
         return {"data": [{"r": "1", "i": "100300", "c": "20"}]}
 
-    def handle_action_getmsg_request(self, data: Dict) -> Dict:
+    async def handle_action_getmsg_request(self, data: Dict) -> Dict:
         return {"msgs": []}
 
-    def handle_auth_logout_request(self, data: Dict) -> Dict:
+    async def handle_auth_logout_request(self, data: Dict) -> Dict:
         return {"auth": True}
 
-    def handle_action_rankreg_request(self, data: Dict) -> Dict:
+    async def handle_action_rankreg_request(self, data: Dict) -> Dict:
         uid = data["rankreg"]["uid"]
         self.logger.info(f"Put {len(data['rankreg']['data'])} rankings for {uid}")
 
@@ -492,7 +492,7 @@ class CxbBase:
             # REV S2
             if "clear" in rid:
                 try:
-                    self.data.score.put_ranking(
+                    await self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
                         song_id=int(rid["sc"][1]),
@@ -500,7 +500,7 @@ class CxbBase:
                         clear=rid["clear"],
                     )
                 except Exception:
-                    self.data.score.put_ranking(
+                    await self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
                         song_id=0,
@@ -510,7 +510,7 @@ class CxbBase:
             # REV
             else:
                 try:
-                    self.data.score.put_ranking(
+                    await self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
                         song_id=int(rid["sc"][1]),
@@ -518,7 +518,7 @@ class CxbBase:
                         clear=0,
                     )
                 except Exception:
-                    self.data.score.put_ranking(
+                    await self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
                         song_id=0,
@@ -527,15 +527,15 @@ class CxbBase:
                     )
         return {}
 
-    def handle_action_addenergy_request(self, data: Dict) -> Dict:
+    async def handle_action_addenergy_request(self, data: Dict) -> Dict:
         uid = data["addenergy"]["uid"]
         self.logger.info(f"Add energy to user {uid}")
-        profile = self.data.profile.get_profile_index(0, uid, self.version)
+        profile = await self.data.profile.get_profile_index(0, uid, self.version)
         data1 = profile["data"]
-        p = self.data.item.get_energy(uid)
+        p = await self.data.item.get_energy(uid)
 
         if not p:
-            self.data.item.put_energy(uid, 5)
+            await self.data.item.put_energy(uid, 5)
 
             return {
                 "class": data1["myClass"],
@@ -548,7 +548,7 @@ class CxbBase:
         energy = p["energy"]
 
         newenergy = int(energy) + 5
-        self.data.item.put_energy(uid, newenergy)
+        await self.data.item.put_energy(uid, newenergy)
 
         if int(energy) <= 995:
             array.append(
@@ -570,10 +570,10 @@ class CxbBase:
             )
         return array[0]
 
-    def handle_action_eventreq_request(self, data: Dict) -> Dict:
+    async def handle_action_eventreq_request(self, data: Dict) -> Dict:
         self.logger.info(data)
         return {"eventreq": ""}
 
-    def handle_action_stampreq_request(self, data: Dict) -> Dict:
+    async def handle_action_stampreq_request(self, data: Dict) -> Dict:
         self.logger.info(data)
         return {"stampreq": ""}

titles/cxb/config.py

@@ -18,6 +18,12 @@ class CxbServerConfig:
                 self.__config, "cxb", "server", "loglevel", default="info"
             )
         )
+    
+    @property
+    def use_https(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "cxb", "server", "use_https", default=True
+        )
 
 
 class CxbConfig(dict):

titles/cxb/data/rss1/Shop/ShopList_Sale.csv

@@ -1,3 +0,0 @@
-saleID.,<2C>J<EFBFBD>n<EFBFBD><6E>,<2C>I<EFBFBD><49><EFBFBD><EFBFBD>,ShopID,Price,    
-0,1411696799,1443236400,0,7000,
-1,1411783199,1443322800,1,7000,

titles/cxb/data/rss1/Shop/ShopList_SkinBg.csv

@@ -1,4 +0,0 @@
-shopID.,整理用No.,Ver.,出現フラグ,出現フラグ参照ID,条件,出現時間,消滅時間,ItemCode,価格,表示タイプ,Text,Type,Value(op),Value,対象曲,Difficulty(op),Difficulty,Level(op),Level,Grade(Op),Grade,GaugeType(op),GaugeType,HS(op)i,HS,APP,DAP,F-V,F-H,FullCombo,Combo(op),Combo,ClearRate(op),ClearRate,プレイ日,地域,
-3000,1,1.00.00,1,-1,-,1411697520.0288,1443233520.0288,skb0000,10,1,MASTER以上の2曲S+以上フルコンボクリアする。,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-3001,2,1.00.00,1,-1,-,1411697520.0288,1443233520.0288,skb0001,10,1,Next Frontier (Master）をクリア,0,-1,-1,bleeze,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-3002,3,1.00.00,1,-1,-,1412103600.0288,1443639598.992,skb0002,10,2,Master以上を1曲をS+以上でクリアする。,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,1,

titles/cxb/data/rss1/Shop/ShopList_SkinEffect.csv

@@ -1,11 +0,0 @@
-shopID.,<2C><><EFBFBD><EFBFBD><EFBFBD>pNo.,Ver.,<2C>o<EFBFBD><6F><EFBFBD>t<EFBFBD><74><EFBFBD>O,<2C>o<EFBFBD><6F><EFBFBD>t<EFBFBD><74><EFBFBD>O<EFBFBD>Q<EFBFBD><51>ID,<2C><><EFBFBD><EFBFBD>,<2C>o<EFBFBD><6F><EFBFBD><EFBFBD><EFBFBD><EFBFBD>,<2C><><EFBFBD>Ŏ<EFBFBD><C58E><EFBFBD>,ItemCode,<2C><><EFBFBD>i,<2C>\<5C><><EFBFBD>^<5E>C<EFBFBD>v,Text,Type,Value(op),Value,<2C>Ώۋ<CE8F>,Difficulty(op),Difficulty,Level(op),Level,Grade(Op),Grade,GaugeType(op),GaugeType,HS(op)i,HS,APP,DAP,F-V,F-H,FullCombo,Combo(op),Combo,ClearRate(op),ClearRate,<2C>v<EFBFBD><76><EFBFBD>C<EFBFBD><43>,<2C>n<EFBFBD><6E>,
-5000,1,10000,1,-1,-,1411697520,1443233520,ske0000,10,1,MASTER<45>ȏ<EFBFBD><C88F>2<EFBFBD><32>S+<2B>ȏ<EFBFBD>t<EFBFBD><74><EFBFBD>R<EFBFBD><52><EFBFBD>{<7B>N<EFBFBD><4E><EFBFBD>A<EFBFBD><41><EFBFBD><EFBFBD>B,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5001,2,10000,1,-1,-,1411697520,1443233520,ske0001,10,1,Next Frontier (Master<65>j<EFBFBD><6A><EFBFBD>N<EFBFBD><4E><EFBFBD>A,0,-1,-1,megaro,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5002,3,10000,1,-1,-,1412103600,1443639598,ske0002,10,2,Master<65>ȏ<EFBFBD><C88F>1<EFBFBD>Ȃ<EFBFBD>S+<2B>ȏ<EFBFBD>ŃN<C583><4E><EFBFBD>A<EFBFBD><41><EFBFBD><EFBFBD>B,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,1,
-5003,4,10000,1,-1,-,1412103600,1443639598,ske0003,10,0,Master<65>ȏ<EFBFBD><C88F>1<EFBFBD>Ȃ<EFBFBD>S+<2B>ȏ<EFBFBD>ŃN<C583><4E><EFBFBD>A<EFBFBD><41><EFBFBD><EFBFBD>B,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5004,5,10000,1,-1,-,1412103600,1443639598,ske0004,10,2,2<>ȃN<C883><4E><EFBFBD>A,1,1,2,-,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5005,5,10000,1,-1,-,1412103600,1443639598,ske0005,10,2,3<>ȃN<C883><4E><EFBFBD>A,1,1,3,-,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5006,5,10000,1,-1,-,1412103600,1443639598,ske0006,10,2,4<>ȃN<C883><4E><EFBFBD>A,1,1,4,-,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5007,5,10000,1,-1,-,1412103600,1443639598,ske0007,10,2,5<>ȃN<C883><4E><EFBFBD>A,1,1,5,-,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5008,5,10000,1,-1,-,1412103600,1443639598,ske0008,10,2,6<>ȃN<C883><4E><EFBFBD>A,1,1,6,-,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-5009,5,10000,1,-1,-,1412103600,1443639598,ske0009,10,2,7<>ȃN<C883><4E><EFBFBD>A,1,1,7,-,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,

titles/cxb/data/rss1/Shop/ShopList_SkinNotes.csv

@@ -1,6 +0,0 @@
-shopID.,<2C><><EFBFBD><EFBFBD><EFBFBD>pNo.,Ver.,<2C>o<EFBFBD><6F><EFBFBD>t<EFBFBD><74><EFBFBD>O,<2C>o<EFBFBD><6F><EFBFBD>t<EFBFBD><74><EFBFBD>O<EFBFBD>Q<EFBFBD><51>ID,<2C><><EFBFBD><EFBFBD>,<2C>o<EFBFBD><6F><EFBFBD><EFBFBD><EFBFBD><EFBFBD>,<2C><><EFBFBD>Ŏ<EFBFBD><C58E><EFBFBD>,ItemCode,<2C><><EFBFBD>i,<2C>\<5C><><EFBFBD>^<5E>C<EFBFBD>v,Text,Type,Value(op),Value,<2C>Ώۋ<CE8F>,Difficulty(op),Difficulty,Level(op),Level,Grade(Op),Grade,GaugeType(op),GaugeType,HS(op)i,HS,APP,DAP,F-V,F-H,FullCombo,Combo(op),Combo,ClearRate(op),ClearRate,<2C>v<EFBFBD><76><EFBFBD>C<EFBFBD><43>,<2C>n<EFBFBD><6E>,
-4000,1,10000,1,-1,-,1411697520,4096483201,skt0000,10,1,MASTER<45>ȏ<EFBFBD><C88F>2<EFBFBD><32>S+<2B>ȏ<EFBFBD>t<EFBFBD><74><EFBFBD>R<EFBFBD><52><EFBFBD>{<7B>N<EFBFBD><4E><EFBFBD>A<EFBFBD><41><EFBFBD><EFBFBD>B,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-4001,2,10000,1,-1,-,1411697520,4096483201,skt0001,10,1,Next Frontier (Master<65>j<EFBFBD><6A><EFBFBD>N<EFBFBD><4E><EFBFBD>A,0,-1,-1,megaro,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-4002,3,10000,1,-1,-,1412103600,4096483201,skt0002,10,2,Master<65>ȏ<EFBFBD><C88F>1<EFBFBD>Ȃ<EFBFBD>S+<2B>ȏ<EFBFBD>ŃN<C583><4E><EFBFBD>A<EFBFBD><41><EFBFBD><EFBFBD>B,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,1,
-4003,4,10000,1,-1,-,1412103600,4096483201,skt0003,10,0,Master<65>ȏ<EFBFBD><C88F>1<EFBFBD>Ȃ<EFBFBD>S+<2B>ȏ<EFBFBD>ŃN<C583><4E><EFBFBD>A<EFBFBD><41><EFBFBD><EFBFBD>B,1,1,1,-,1,2,-1,-1,1,1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,
-4004,5,10000,1,-1,-,1412103600,4096483201,skt0004,10,2,aaaaaaaaaaaaaaaaa,1,1,20,-,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,,,

titles/cxb/index.py

@@ -1,4 +1,6 @@
-from twisted.web.http import Request
+from starlette.requests import Request
+from starlette.routing import Route
+from starlette.responses import Response, JSONResponse
 import traceback
 import sys
 import yaml
@@ -11,7 +13,7 @@ from typing import Dict, Tuple, List
 from os import path
 
 from core.config import CoreConfig
-from core.title import BaseServlet
+from core.title import BaseServlet, JSONResponseNoASCII
 from core.utils import Utils
 from .config import CxbConfig
 from .const import CxbConstants
@@ -62,6 +64,14 @@ class CxbServlet(BaseServlet):
             CxbRevSunriseS2(core_cfg, self.game_cfg),
         ]
 
+    def get_routes(self) -> List[Route]:
+        return [
+            Route("/data", self.handle_data, methods=['POST']),
+            Route("/action", self.handle_action, methods=['POST']),
+            Route("/v2/action", self.handle_action, methods=['POST']),
+            Route("/auth", self.handle_auth, methods=['POST']),
+        ]
+
     @classmethod
     def is_game_enabled(cls, game_code: str, core_cfg: CoreConfig, cfg_dir: str) -> bool:
         game_cfg = CxbConfig()
@@ -79,35 +89,22 @@ class CxbServlet(BaseServlet):
         title_port_int = Utils.get_title_port(self.core_cfg)
         title_port_ssl_int = Utils.get_title_port_ssl(self.core_cfg)
     
-        proto = "https" if title_port_ssl_int != 443 else "http"
+        proto = "https" if self.game_cfg.server.use_https else "http"
 
         if proto == "https":
-            t_port = f":{title_port_ssl_int}" if title_port_ssl_int and not self.core_cfg.server.is_using_proxy else ""
+            t_port = f":{title_port_ssl_int}" if title_port_ssl_int != 443 else ""
         
         else:    
-            t_port = f":{title_port_int}" if title_port_int and not self.core_cfg.server.is_using_proxy else ""
+            t_port = f":{title_port_int}" if title_port_int != 80 else ""
 
         return (
-            f"{proto}://{self.core_cfg.title.hostname}{t_port}",
+            f"{proto}://{self.core_cfg.server.hostname}{t_port}",
             "",
         )
-    
-    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
-        return (
-            [], 
-            [
-                ("handle_data", "/data", {}),
-                ("handle_action", "/action", {}),
-                ("handle_action", "/v2/action", {}),
-                ("handle_auth", "/auth", {}),
-            ]
-        )
 
-    def preprocess(self, req: Request) -> Dict:
-        try:
-            req_bytes = req.content.getvalue()
-        except:
-            req_bytes = req.content.read() # Can we just use this one?
+    
+    async def preprocess(self, req: Request) -> Dict:
+        req_bytes = await req.body()
         
         try:
             req_json: Dict = json.loads(req_bytes)
@@ -126,8 +123,8 @@ class CxbServlet(BaseServlet):
         
         return req_json
 
-    def handle_data(self, request: Request, game_code: str, matchers: Dict) -> bytes:
-        req_json = self.preprocess(request)
+    async def handle_data(self, request: Request) -> bytes:
+        req_json = await self.preprocess(request)
         func_to_find = "handle_data_"
         version_string = "Base"
         internal_ver = 0
@@ -135,7 +132,7 @@ class CxbServlet(BaseServlet):
         
         if req_json == {}:
             self.logger.warning(f"Empty json request to /data")
-            return b""
+            return Response()
 
         subcmd = list(req_json.keys())[0]
         if subcmd == "dldate":
@@ -145,14 +142,14 @@ class CxbServlet(BaseServlet):
                 or "filetype" not in req_json["dldate"]
             ):
                 self.logger.warning(f"Malformed dldate request: {req_json}")
-                return b""
+                return Response()
 
             filetype = req_json["dldate"]["filetype"]
             filetype_split = filetype.split("/")
 
             if len(filetype_split) < 2 or not filetype_split[0].isnumeric():
                 self.logger.warning(f"Malformed dldate request: {req_json}")
-                return b""
+                return Response()
 
             version = int(filetype_split[0])
             filename = filetype_split[len(filetype_split) - 1]
@@ -184,7 +181,7 @@ class CxbServlet(BaseServlet):
 
         if not hasattr(self.versions[internal_ver], func_to_find):
             self.logger.warn(f"{version_string} has no handler for filetype {filetype} / {func_to_find}")
-            return({"data":""})
+            return JSONResponse({"data":""})
         
         self.logger.info(f"{version_string} request for filetype {filetype}")
         self.logger.debug(req_json)
@@ -192,7 +189,7 @@ class CxbServlet(BaseServlet):
         handler = getattr(self.versions[internal_ver], func_to_find)
         
         try:
-            resp = handler(req_json)
+            resp = await handler(req_json)
 
         except Exception as e:
             self.logger.error(f"Error handling request for file {filetype} - {e}")
@@ -201,19 +198,19 @@ class CxbServlet(BaseServlet):
                 traceback.print_exception(tp, val, tb, limit=1)
                 with open("{0}/{1}.log".format(self.core_cfg.server.log_dir, "cxb"), "a") as f:
                     traceback.print_exception(tp, val, tb, limit=1, file=f)
-            return ""
+            return Response()
         
         self.logger.debug(f"{version_string} Response {resp}")
-        return json.dumps(resp, ensure_ascii=False).encode("utf-8")
+        return JSONResponseNoASCII(resp)
 
-    def handle_action(self, request: Request, game_code: str, matchers: Dict) -> bytes:
-        req_json = self.preprocess(request)
+    async def handle_action(self, request: Request) -> bytes:
+        req_json = await self.preprocess(request)
         subcmd = list(req_json.keys())[0]
         func_to_find = f"handle_action_{subcmd}_request"
         
         if not hasattr(self.versions[0], func_to_find):
             self.logger.warn(f"No handler for action {subcmd} request")
-            return ""
+            return Response()
         
         self.logger.info(f"Action {subcmd} Request")
         self.logger.debug(req_json)
@@ -221,7 +218,7 @@ class CxbServlet(BaseServlet):
         handler = getattr(self.versions[0], func_to_find)
         
         try:
-            resp = handler(req_json)
+            resp = await handler(req_json)
 
         except Exception as e:
             self.logger.error(f"Error handling action {subcmd} request - {e}")
@@ -230,19 +227,19 @@ class CxbServlet(BaseServlet):
                 traceback.print_exception(tp, val, tb, limit=1)
                 with open("{0}/{1}.log".format(self.core_cfg.server.log_dir, "cxb"), "a") as f:
                     traceback.print_exception(tp, val, tb, limit=1, file=f)
-            return ""
+            return Response()
         
         self.logger.debug(f"Response {resp}")
-        return json.dumps(resp, ensure_ascii=False).encode("utf-8")
+        return JSONResponseNoASCII(resp)
 
-    def handle_auth(self, request: Request, game_code: str, matchers: Dict) -> bytes:
-        req_json = self.preprocess(request)
+    async def handle_auth(self, request: Request) -> bytes:
+        req_json = await self.preprocess(request)
         subcmd = list(req_json.keys())[0]
         func_to_find = f"handle_auth_{subcmd}_request"
         
         if not hasattr(self.versions[0], func_to_find):
             self.logger.warn(f"No handler for auth {subcmd} request")
-            return ""
+            return Response()
         
         self.logger.info(f"Action {subcmd} Request")
         self.logger.debug(req_json)
@@ -250,7 +247,7 @@ class CxbServlet(BaseServlet):
         handler = getattr(self.versions[0], func_to_find)
         
         try:
-            resp = handler(req_json)
+            resp = await handler(req_json)
 
         except Exception as e:
             self.logger.error(f"Error handling auth {subcmd} request - {e}")
@@ -259,7 +256,7 @@ class CxbServlet(BaseServlet):
                 traceback.print_exception(tp, val, tb, limit=1)
                 with open("{0}/{1}.log".format(self.core_cfg.server.log_dir, "cxb"), "a") as f:
                     traceback.print_exception(tp, val, tb, limit=1, file=f)
-            return ""
+            return Response()
         
         self.logger.debug(f"Response {resp}")
-        return json.dumps(resp, ensure_ascii=False).encode("utf-8")
+        return JSONResponseNoASCII(resp)

titles/cxb/read.py

@@ -1,6 +1,5 @@
-from typing import Optional, Dict, List
-from os import walk, path
-import urllib
+from typing import Optional
+from os import path
 import csv
 
 from read import BaseReader
@@ -8,7 +7,6 @@ from core.config import CoreConfig
 from titles.cxb.database import CxbData
 from titles.cxb.const import CxbConstants
 
-
 class CxbReader(BaseReader):
     def __init__(
         self,
@@ -29,17 +27,14 @@ class CxbReader(BaseReader):
             self.logger.error(f"Invalid project cxb version {version}")
             exit(1)
 
-    def read(self) -> None:
-        pull_bin_ram = True
+    async def read(self) -> None:
+        if path.exists(self.bin_dir):
+            await self.read_csv(self.bin_dir)
+        
+        else:
+            self.logger.warn(f"{self.bin_dir} does not exist, nothing to import")
 
-        if not path.exists(f"{self.bin_dir}"):
-            self.logger.warning(f"Couldn't find csv file in {self.bin_dir}, skipping")
-            pull_bin_ram = False
-
-        if pull_bin_ram:
-            self.read_csv(f"{self.bin_dir}")
-
-    def read_csv(self, bin_dir: str) -> None:
+    async def read_csv(self, bin_dir: str) -> None:
         self.logger.info(f"Read csv from {bin_dir}")
 
         try:
@@ -55,7 +50,7 @@ class CxbReader(BaseReader):
 
                     if not "N/A" in row["standard"]:
                         self.logger.info(f"Added song {song_id} chart 0")
-                        self.data.static.put_music(
+                        await self.data.static.put_music(
                             self.version,
                             song_id,
                             index,
@@ -71,7 +66,7 @@ class CxbReader(BaseReader):
                         )
                     if not "N/A" in row["hard"]:
                         self.logger.info(f"Added song {song_id} chart 1")
-                        self.data.static.put_music(
+                        await self.data.static.put_music(
                             self.version,
                             song_id,
                             index,
@@ -83,7 +78,7 @@ class CxbReader(BaseReader):
                         )
                     if not "N/A" in row["master"]:
                         self.logger.info(f"Added song {song_id} chart 2")
-                        self.data.static.put_music(
+                        await self.data.static.put_music(
                             self.version,
                             song_id,
                             index,
@@ -97,7 +92,7 @@ class CxbReader(BaseReader):
                         )
                     if not "N/A" in row["unlimited"]:
                         self.logger.info(f"Added song {song_id} chart 3")
-                        self.data.static.put_music(
+                        await self.data.static.put_music(
                             self.version,
                             song_id,
                             index,
@@ -113,7 +108,7 @@ class CxbReader(BaseReader):
                         )
                     if not "N/A" in row["easy"]:
                         self.logger.info(f"Added song {song_id} chart 4")
-                        self.data.static.put_music(
+                        await self.data.static.put_music(
                             self.version,
                             song_id,
                             index,

titles/cxb/rev.py

@@ -17,15 +17,15 @@ class CxbRev(CxbBase):
         super().__init__(cfg, game_cfg)
         self.version = CxbConstants.VER_CROSSBEATS_REV
 
-    def handle_data_path_list_request(self, data: Dict) -> Dict:
+    async def handle_data_path_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_putlog_request(self, data: Dict) -> Dict:
+    async def handle_data_putlog_request(self, data: Dict) -> Dict:
         if data["putlog"]["type"] == "ResultLog":
             score_data = json.loads(data["putlog"]["data"])
             userid = score_data["usid"]
 
-            self.data.score.put_playlog(
+            await self.data.score.put_playlog(
                 userid,
                 score_data["mcode"],
                 score_data["difficulty"],
@@ -45,9 +45,9 @@ class CxbRev(CxbBase):
         return {"data": True}
 
     @cached(lifetime=86400)
-    def handle_data_music_list_request(self, data: Dict) -> Dict:
+    async def handle_data_music_list_request(self, data: Dict) -> Dict:
         ret_str = ""
-        with open(r"titles/cxb/data/rss/MusicArchiveList.csv") as music:
+        with open(r"titles/cxb/data/rev/MusicArchiveList.csv") as music:
             lines = music.readlines()
             for line in lines:
                 line_split = line.split(",")
@@ -56,10 +56,10 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_item_list_icon_request(self, data: Dict) -> Dict:
+    async def handle_data_item_list_icon_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ItemListIcon\r\n"
         with open(
-            r"titles/cxb/data/rss/Item/ItemArchiveList_Icon.csv", encoding="utf-8"
+            r"titles/cxb/data/rev/Item/ItemArchiveList_Icon.csv", encoding="shift-jis"
         ) as item:
             lines = item.readlines()
             for line in lines:
@@ -67,10 +67,10 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_item_list_skin_notes_request(self, data: Dict) -> Dict:
+    async def handle_data_item_list_skin_notes_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ItemListSkinNotes\r\n"
         with open(
-            r"titles/cxb/data/rss/Item/ItemArchiveList_SkinNotes.csv", encoding="utf-8"
+            r"titles/cxb/data/rev/Item/ItemArchiveList_SkinNotes.csv", encoding="utf-8"
         ) as item:
             lines = item.readlines()
             for line in lines:
@@ -78,10 +78,10 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_item_list_skin_effect_request(self, data: Dict) -> Dict:
+    async def handle_data_item_list_skin_effect_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ItemListSkinEffect\r\n"
         with open(
-            r"titles/cxb/data/rss/Item/ItemArchiveList_SkinEffect.csv", encoding="utf-8"
+            r"titles/cxb/data/rev/Item/ItemArchiveList_SkinEffect.csv", encoding="utf-8"
         ) as item:
             lines = item.readlines()
             for line in lines:
@@ -89,10 +89,10 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_item_list_skin_bg_request(self, data: Dict) -> Dict:
+    async def handle_data_item_list_skin_bg_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ItemListSkinBg\r\n"
         with open(
-            r"titles/cxb/data/rss/Item/ItemArchiveList_SkinBg.csv", encoding="utf-8"
+            r"titles/cxb/data/rev/Item/ItemArchiveList_SkinBg.csv", encoding="utf-8"
         ) as item:
             lines = item.readlines()
             for line in lines:
@@ -100,10 +100,10 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_item_list_title_request(self, data: Dict) -> Dict:
+    async def handle_data_item_list_title_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ItemListTitle\r\n"
         with open(
-            r"titles/cxb/data/rss/Item/ItemList_Title.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/Item/ItemList_Title.csv", encoding="shift-jis"
         ) as item:
             lines = item.readlines()
             for line in lines:
@@ -111,10 +111,10 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_shop_list_music_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_music_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ShopListMusic\r\n"
         with open(
-            r"titles/cxb/data/rss/Shop/ShopList_Music.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/Shop/ShopList_Music.csv", encoding="shift-jis"
         ) as shop:
             lines = shop.readlines()
             for line in lines:
@@ -122,10 +122,10 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_shop_list_icon_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_icon_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ShopListIcon\r\n"
         with open(
-            r"titles/cxb/data/rss/Shop/ShopList_Icon.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/Shop/ShopList_Icon.csv", encoding="shift-jis"
         ) as shop:
             lines = shop.readlines()
             for line in lines:
@@ -133,83 +133,88 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_shop_list_title_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_title_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ShopListTitle\r\n"
         with open(
-            r"titles/cxb/data/rss/Shop/ShopList_Title.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/Shop/ShopList_Title.csv", encoding="shift-jis"
         ) as shop:
             lines = shop.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_shop_list_skin_hud_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_skin_hud_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_shop_list_skin_arrow_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_skin_arrow_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_shop_list_skin_hit_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_skin_hit_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_shop_list_sale_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_sale_request(self, data: Dict) -> Dict:
         ret_str = "\r\n#ShopListSale\r\n"
         with open(
-            r"titles/cxb/data/rss/Shop/ShopList_Sale.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/Shop/ShopList_Sale.csv", encoding="shift-jis"
         ) as shop:
             lines = shop.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_extra_stage_list_request(self, data: Dict) -> Dict:
-        return {"data": ""}
+    async def handle_data_extra_stage_list_request(self, data: Dict) -> Dict:
+        ret_str = ""
+        with open(r"titles/cxb/data/rev/ExtraStageList.csv", encoding="shift-jis") as stage:
+            lines = stage.readlines()
+            for line in lines:
+                ret_str += f"{line[:-1]}\r\n"
+        return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_exxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_exxxxx_request(self, data: Dict) -> Dict:
         extra_num = int(data["dldate"]["filetype"][-4:])
         ret_str = ""
         with open(
-            rf"titles/cxb/data/rss/Ex000{extra_num}.csv", encoding="shift-jis"
+            rf"titles/cxb/data/rev/Ex000{extra_num}.csv", encoding="shift-jis"
         ) as stage:
             lines = stage.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_bonus_list10100_request(self, data: Dict) -> Dict:
+    async def handle_data_bonus_list10100_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_free_coupon_request(self, data: Dict) -> Dict:
+    async def handle_data_free_coupon_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_news_list_request(self, data: Dict) -> Dict:
+    async def handle_data_news_list_request(self, data: Dict) -> Dict:
         ret_str = ""
-        with open(r"titles/cxb/data/rss/NewsList.csv", encoding="UTF-8") as news:
+        with open(r"titles/cxb/data/rev/NewsList.csv", encoding="UTF-8") as news:
             lines = news.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_tips_request(self, data: Dict) -> Dict:
+    async def handle_data_tips_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_license_request(self, data: Dict) -> Dict:
+    async def handle_data_license_request(self, data: Dict) -> Dict:
         ret_str = ""
-        with open(r"titles/cxb/data/rss/License_Offline.csv", encoding="UTF-8") as lic:
+        with open(r"titles/cxb/data/rev/License_Offline.csv", encoding="UTF-8") as lic:
             lines = lic.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_course_list_request(self, data: Dict) -> Dict:
+    async def handle_data_course_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(
-            r"titles/cxb/data/rss/Course/CourseList.csv", encoding="UTF-8"
+            r"titles/cxb/data/rev/Course/CourseList.csv", encoding="UTF-8"
         ) as course:
             lines = course.readlines()
             for line in lines:
@@ -217,12 +222,12 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_csxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_csxxxx_request(self, data: Dict) -> Dict:
         # Removed the CSVs since the format isnt quite right
         extra_num = int(data["dldate"]["filetype"][-4:])
         ret_str = ""
         with open(
-            rf"titles/cxb/data/rss/Course/Cs000{extra_num}.csv", encoding="shift-jis"
+            rf"titles/cxb/data/rev/Course/Cs000{extra_num}.csv", encoding="shift-jis"
         ) as course:
             lines = course.readlines()
             for line in lines:
@@ -230,77 +235,77 @@ class CxbRev(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_mission_list_request(self, data: Dict) -> Dict:
+    async def handle_data_mission_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(
-            r"titles/cxb/data/rss/MissionList.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/MissionList.csv", encoding="shift-jis"
         ) as mission:
             lines = mission.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_mission_bonus_request(self, data: Dict) -> Dict:
+    async def handle_data_mission_bonus_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_unlimited_mission_request(self, data: Dict) -> Dict:
+    async def handle_data_unlimited_mission_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_event_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(
-            r"titles/cxb/data/rss/Event/EventArchiveList.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/Event/EventArchiveList.csv", encoding="shift-jis"
         ) as mission:
             lines = mission.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_event_music_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_music_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_mission_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_mission_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_achievement_single_high_score_list_request(
+    async def handle_data_event_achievement_single_high_score_list_request(
         self, data: Dict
     ) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_achievement_single_accumulation_request(
+    async def handle_data_event_achievement_single_accumulation_request(
         self, data: Dict
     ) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_ranking_high_score_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_ranking_high_score_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_ranking_accumulation_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_ranking_accumulation_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_ranking_stamp_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_ranking_stamp_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_ranking_store_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_ranking_store_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_ranking_area_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_ranking_area_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_event_stamp_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_stamp_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(
-            r"titles/cxb/data/rss/Event/EventStampList.csv", encoding="shift-jis"
+            r"titles/cxb/data/rev/Event/EventStampList.csv", encoding="shift-jis"
         ) as event:
             lines = event.readlines()
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_event_stamp_map_list_csxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_event_stamp_map_list_csxxxx_request(self, data: Dict) -> Dict:
         return {"data": "1,2,1,1,2,3,9,5,6,7,8,9,10,\r\n"}
 
-    def handle_data_server_state_request(self, data: Dict) -> Dict:
+    async def handle_data_server_state_request(self, data: Dict) -> Dict:
         return {"data": True}

titles/cxb/rss1.py

@@ -17,11 +17,11 @@ class CxbRevSunriseS1(CxbBase):
         super().__init__(cfg, game_cfg)
         self.version = CxbConstants.VER_CROSSBEATS_REV_SUNRISE_S1
 
-    def handle_data_path_list_request(self, data: Dict) -> Dict:
+    async def handle_data_path_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_music_list_request(self, data: Dict) -> Dict:
+    async def handle_data_music_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(r"titles/cxb/data/rss1/MusicArchiveList.csv") as music:
             lines = music.readlines()
@@ -32,7 +32,7 @@ class CxbRevSunriseS1(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_item_list_detail_request(self, data: Dict) -> Dict:
+    async def handle_data_item_list_detail_request(self, data: Dict) -> Dict:
         # ItemListIcon load
         ret_str = "#ItemListIcon\r\n"
         with open(
@@ -54,7 +54,7 @@ class CxbRevSunriseS1(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_shop_list_detail_request(self, data: Dict) -> Dict:
+    async def handle_data_shop_list_detail_request(self, data: Dict) -> Dict:
         # ShopListIcon load
         ret_str = "#ShopListIcon\r\n"
         with open(
@@ -73,42 +73,6 @@ class CxbRevSunriseS1(CxbBase):
             for line in lines:
                 ret_str += f"{line[:-1]}\r\n"
 
-        # ShopListSale load
-        ret_str += "\r\n#ShopListSale\r\n"
-        with open(
-            r"titles/cxb/data/rss1/Shop/ShopList_Sale.csv", encoding="shift-jis"
-        ) as shop:
-            lines = shop.readlines()
-            for line in lines:
-                ret_str += f"{line[:-1]}\r\n"
-
-        # ShopListSkinBg load
-        ret_str += "\r\n#ShopListSkinBg\r\n"
-        with open(
-            r"titles/cxb/data/rss1/Shop/ShopList_SkinBg.csv", encoding="shift-jis"
-        ) as shop:
-            lines = shop.readlines()
-            for line in lines:
-                ret_str += f"{line[:-1]}\r\n"
-
-        # ShopListSkinEffect load
-        ret_str += "\r\n#ShopListSkinEffect\r\n"
-        with open(
-            r"titles/cxb/data/rss1/Shop/ShopList_SkinEffect.csv", encoding="shift-jis"
-        ) as shop:
-            lines = shop.readlines()
-            for line in lines:
-                ret_str += f"{line[:-1]}\r\n"
-
-        # ShopListSkinNotes load
-        ret_str += "\r\n#ShopListSkinNotes\r\n"
-        with open(
-            r"titles/cxb/data/rss1/Shop/ShopList_SkinNotes.csv", encoding="shift-jis"
-        ) as shop:
-            lines = shop.readlines()
-            for line in lines:
-                ret_str += f"{line[:-1]}\r\n"
-
         # ShopListTitle load
         ret_str += "\r\n#ShopListTitle\r\n"
         with open(
@@ -119,26 +83,26 @@ class CxbRevSunriseS1(CxbBase):
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_extra_stage_list_request(self, data: Dict) -> Dict:
+    async def handle_data_extra_stage_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_exxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_exxxxx_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_one_more_extra_list_request(self, data: Dict) -> Dict:
+    async def handle_data_one_more_extra_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_bonus_list10100_request(self, data: Dict) -> Dict:
+    async def handle_data_bonus_list10100_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_oexxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_oexxxx_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_free_coupon_request(self, data: Dict) -> Dict:
+    async def handle_data_free_coupon_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_news_list_request(self, data: Dict) -> Dict:
+    async def handle_data_news_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(r"titles/cxb/data/rss1/NewsList.csv", encoding="UTF-8") as news:
             lines = news.readlines()
@@ -146,14 +110,14 @@ class CxbRevSunriseS1(CxbBase):
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_tips_request(self, data: Dict) -> Dict:
+    async def handle_data_tips_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_release_info_list_request(self, data: Dict) -> Dict:
+    async def handle_data_release_info_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
     @cached(lifetime=86400)
-    def handle_data_random_music_list_request(self, data: Dict) -> Dict:
+    async def handle_data_random_music_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(r"titles/cxb/data/rss1/MusicArchiveList.csv") as music:
             lines = music.readlines()
@@ -167,7 +131,7 @@ class CxbRevSunriseS1(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_license_request(self, data: Dict) -> Dict:
+    async def handle_data_license_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(r"titles/cxb/data/rss1/License.csv", encoding="UTF-8") as licenses:
             lines = licenses.readlines()
@@ -176,7 +140,7 @@ class CxbRevSunriseS1(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_course_list_request(self, data: Dict) -> Dict:
+    async def handle_data_course_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         with open(
             r"titles/cxb/data/rss1/Course/CourseList.csv", encoding="UTF-8"
@@ -187,7 +151,7 @@ class CxbRevSunriseS1(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_csxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_csxxxx_request(self, data: Dict) -> Dict:
         extra_num = int(data["dldate"]["filetype"][-4:])
         ret_str = ""
         with open(
@@ -198,16 +162,16 @@ class CxbRevSunriseS1(CxbBase):
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_mission_list_request(self, data: Dict) -> Dict:
+    async def handle_data_mission_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_mission_bonus_request(self, data: Dict) -> Dict:
+    async def handle_data_mission_bonus_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_unlimited_mission_request(self, data: Dict) -> Dict:
+    async def handle_data_unlimited_mission_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_partner_list_request(self, data: Dict) -> Dict:
+    async def handle_data_partner_list_request(self, data: Dict) -> Dict:
         ret_str = ""
         # Lord forgive me for the sins I am about to commit
         for i in range(0, 10):
@@ -226,7 +190,7 @@ class CxbRevSunriseS1(CxbBase):
         return {"data": ret_str}
 
     @cached(lifetime=86400)
-    def handle_data_partnerxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_partnerxxxx_request(self, data: Dict) -> Dict:
         partner_num = int(data["dldate"]["filetype"][-4:])
         ret_str = f"{partner_num},,{partner_num},1,10000,\r\n"
         with open(r"titles/cxb/data/rss1/Partner0000.csv") as partner:
@@ -235,13 +199,13 @@ class CxbRevSunriseS1(CxbBase):
                 ret_str += f"{line[:-1]}\r\n"
         return {"data": ret_str}
 
-    def handle_data_server_state_request(self, data: Dict) -> Dict:
+    async def handle_data_server_state_request(self, data: Dict) -> Dict:
         return {"data": True}
 
-    def handle_data_settings_request(self, data: Dict) -> Dict:
+    async def handle_data_settings_request(self, data: Dict) -> Dict:
         return {"data": "2,\r\n"}
 
-    def handle_data_story_list_request(self, data: Dict) -> Dict:
+    async def handle_data_story_list_request(self, data: Dict) -> Dict:
         # story id, story name, game version, start time, end time, course arc, unlock flag, song mcode for menu
         ret_str = "\r\n"
         ret_str += (
@@ -253,23 +217,23 @@ class CxbRevSunriseS1(CxbBase):
         ret_str += f"st0002,REMNANT,10104,1502127790,4096483201,Cs1000,-1,overcl,\r\n"
         return {"data": ret_str}
 
-    def handle_data_stxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_stxxxx_request(self, data: Dict) -> Dict:
         story_num = int(data["dldate"]["filetype"][-4:])
         ret_str = ""
         for i in range(1, 11):
             ret_str += f"{i},st000{story_num}_{i-1},,,,,,,,,,,,,,,,1,,-1,1,\r\n"
         return {"data": ret_str}
 
-    def handle_data_event_stamp_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_stamp_list_request(self, data: Dict) -> Dict:
         return {"data": "Cs1032,1,1,1,1,1,1,1,1,1,1,\r\n"}
 
-    def handle_data_premium_list_request(self, data: Dict) -> Dict:
+    async def handle_data_premium_list_request(self, data: Dict) -> Dict:
         return {"data": "1,,,,10,,,,,99,,,,,,,,,100,,\r\n"}
 
-    def handle_data_event_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_list_request(self, data: Dict) -> Dict:
         return {"data": ""}
 
-    def handle_data_event_detail_list_request(self, data: Dict) -> Dict:
+    async def handle_data_event_detail_list_request(self, data: Dict) -> Dict:
         event_id = data["dldate"]["filetype"].split("/")[2]
         if "EventStampMapListCs1002" in event_id:
             return {"data": "1,2,1,1,2,3,9,5,6,7,8,9,10,\r\n"}
@@ -278,7 +242,7 @@ class CxbRevSunriseS1(CxbBase):
         else:
             return {"data": ""}
 
-    def handle_data_event_stamp_map_list_csxxxx_request(self, data: Dict) -> Dict:
+    async def handle_data_event_stamp_map_list_csxxxx_request(self, data: Dict) -> Dict:
         event_id = data["dldate"]["filetype"].split("/")[2]
         if "EventStampMapListCs1002" in event_id:
             return {"data": "1,2,1,1,2,3,9,5,6,7,8,9,10,\r\n"}