experiment

Reviewed-on: Hay1tsme/artemis#78

.gitattributes

@@ -0,0 +1,3 @@
+*.csv binary
+*.txt binary
+*.json binary

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.9.15-slim-bullseye
+
+RUN apt update && apt install default-libmysqlclient-dev build-essential libtk nodejs npm pkg-config -y
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+RUN npm i -g nodemon
+
+COPY entrypoint.sh entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+COPY index.py index.py
+COPY dbutils.py dbutils.py
+COPY read.py read.py
+ADD core core
+ADD titles titles
+ADD config config
+ADD logs logs
+ADD cert cert
+
+ENTRYPOINT [ "/app/entrypoint.sh" ]

alembic.ini

@@ -0,0 +1,116 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = mysql://root:root@localhost/artemis2023?charset=utf8mb4
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = [%(asctime)s] %(levelname)s | %(name)s | %(message)s
+datefmt = %Y-%m-%d %H:%M:%S

alembic/env.py

@@ -0,0 +1,111 @@
+from logging.config import fileConfig
+from os import path
+
+import yaml
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+from core import CoreConfig
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name, disable_existing_loggers=False)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+
+# Core
+from core.data.schema import metadata
+
+# Titles
+from titles.chuni.schema import *
+from titles.cm.schema import *
+from titles.cxb.schema.item import *
+from titles.diva.schema import *
+from titles.idac.schema import *
+# from titles.idz.schema import *
+from titles.mai2.schema import *
+from titles.ongeki.schema import *
+from titles.pokken.schema import *
+from titles.sao.schema import *
+
+target_metadata = metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def get_url_from_config():
+    url = config.get_main_option('artemis_db_url')
+    if url is not None:
+        return url
+
+    import core.data.database
+
+    cfg: CoreConfig = CoreConfig()
+    if path.exists(f"config/core.yaml"):
+        cfg.update(yaml.safe_load(open(f"config/core.yaml")))
+    return core.data.database.build_db_url_from_config(cfg.database)
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    # Load config
+    url = get_url_from_config()
+
+    from sqlalchemy import create_engine
+    engine = create_engine(url)
+    with engine.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

changelog.md

@@ -1,6 +1,75 @@
 # Changelog
 Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
 
+## 20231015
+### maimai DX
++ Added support for FESTiVAL PLUS
+
+### Card Maker
++ Added support for maimai DX FESTiVAL PLUS
+
+## 20231001
+### Initial D THE ARCADE
++ Added support for Initial D THE ARCADE S2
+  + Story mode progress added
+  + Bunta Challenge/Touhou Project modes added
+  + Time Trials added
+  + Leaderboards added, but doesn't refresh sometimes
+  + Theory of Street mode added (with CPUs)
+  + Play Stamp/Timetrial events added
+  + Frontend to download profile added
+  + Importer to import profiles added
+
+## 20230716
+### General
++ Docker files added (#19)
++ Added support for threading
+  + This comes with the caviat that enabling it will not allow you to use Ctrl + C to stop the server.
+
+### Webui
++ Small improvements
++ Add card display
+
+### Allnet
++ Billing format validation
++ Fix naomitest.html endpoint
++ Add event logging for auths and billing
++ LoaderStateRecorder endpoint handler added
+
+### Mucha
++ Fixed log level always being "Info"
++ Add stub handler for DownloadState
+
+### Sword Art Online
++ Support added
+
+### Crossbeats
++ Added threading to profile loading
+  + This should cause a noticeable speed-up
+
+### Card Maker
++ DX Passes fixed
++ Various improvements
+
+### Diva
++ Added clear status calculation
++ Various minor fixes and improvements
+
+### Maimai
++ Added support for memorial photo uploads
++ Added support for the following versions
+  + Festival
+  + FiNALE
++ Various bug fixes and improvements
+
+### Wacca
++ Fixed an error that sometimes occoured when trying to unlock songs (#22)
+
+### Pokken
++ Profile saving added (loading TBA)
++ Use external STUN server for matching by default
+  + Matching still not working
+
 ## 2023042300
 ### Wacca
 + Time free now works properly

core/adb_handlers/__init__.py

@@ -0,0 +1,6 @@
+from .base import ADBBaseRequest, ADBBaseResponse, ADBHeader, ADBHeaderException, PortalRegStatus, LogStatus, ADBStatus
+from .base import CompanyCodes, ReaderFwVer, CMD_CODE_GOODBYE, HEADER_SIZE
+from .lookup import ADBLookupRequest, ADBLookupResponse, ADBLookupExResponse
+from .campaign import ADBCampaignClearRequest, ADBCampaignClearResponse, ADBCampaignResponse, ADBOldCampaignRequest, ADBOldCampaignResponse
+from .felica import ADBFelicaLookupRequest, ADBFelicaLookupResponse, ADBFelicaLookup2Request, ADBFelicaLookup2Response
+from .log import ADBLogExRequest, ADBLogRequest, ADBStatusLogRequest, ADBLogExResponse

core/adb_handlers/base.py

@@ -0,0 +1,170 @@
+import struct
+from construct import Struct, Int16ul, Int32ul, PaddedString
+from enum import Enum
+import re
+from typing import Union, Final
+
+class LogStatus(Enum):
+    NONE = 0
+    START = 1
+    CONTINUE = 2
+    END = 3
+    OTHER = 4
+
+class PortalRegStatus(Enum):
+    NO_REG = 0
+    PORTAL = 1
+    SEGA_ID = 2
+
+class ADBStatus(Enum):
+    UNKNOWN = 0
+    GOOD = 1
+    BAD_AMIE_ID = 2
+    ALREADY_REG = 3
+    BAN_SYS_USER = 4
+    BAN_SYS = 5
+    BAN_USER = 6
+    BAN_GEN = 7
+    LOCK_SYS_USER = 8
+    LOCK_SYS = 9
+    LOCK_USER = 10
+
+class CompanyCodes(Enum):
+    NONE = 0
+    SEGA = 1
+    BAMCO = 2
+    KONAMI = 3
+    TAITO = 4
+
+class ReaderFwVer(Enum): # Newer readers use a singly byte value
+    NONE = 0
+    TN32_10 = 1
+    TN32_12 = 2
+    OTHER = 9
+    
+    def __str__(self) -> str:
+        if self == self.TN32_10:
+            return "TN32MSEC003S F/W Ver1.0"
+        elif self == self.TN32_12:
+            return "TN32MSEC003S F/W Ver1.2"
+        elif self == self.NONE:
+            return "Not Specified"
+        elif self == self.OTHER:
+            return "Unknown/Other"
+        else:
+            raise ValueError(f"Bad ReaderFwVer value {self.value}")
+    
+    @classmethod
+    def from_byte(self, byte: bytes) -> Union["ReaderFwVer", int]:
+        try:
+            i = int.from_bytes(byte, 'little')
+            try:
+                return ReaderFwVer(i)
+            except ValueError:
+                return i
+        except TypeError:
+            return 0
+
+class ADBHeaderException(Exception):
+    pass
+
+HEADER_SIZE: Final[int] = 0x20
+CMD_CODE_GOODBYE: Final[int] = 0x66
+
+# everything is LE
+class ADBHeader:
+    def __init__(self, magic: int, protocol_ver: int, cmd: int, length: int, status: int, game_id: Union[str, bytes], store_id: int, keychip_id: Union[str, bytes]) -> None:
+        self.magic = magic # u16
+        self.protocol_ver = protocol_ver # u16
+        self.cmd = cmd # u16
+        self.length = length # u16
+        try:
+            self.status = ADBStatus(status) # u16
+        except ValueError as e:
+            raise ADBHeaderException(f"Status is incorrect! {e}")
+        self.game_id = game_id # 4 char + \x00
+        self.store_id = store_id # u32
+        self.keychip_id = keychip_id# 11 char + \x00
+                
+        if type(self.game_id) == bytes:
+            self.game_id = self.game_id.decode()
+        
+        if type(self.keychip_id) == bytes:
+            self.keychip_id = self.keychip_id.decode()
+        
+        self.game_id = self.game_id.replace("\0", "")
+        self.keychip_id = self.keychip_id.replace("\0", "")
+        if self.cmd != CMD_CODE_GOODBYE: # Games for some reason send no data with goodbye
+            self.validate()
+    
+    @classmethod
+    def from_data(cls, data: bytes) -> "ADBHeader":
+        magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id = struct.unpack_from("<5H6sI12s", data)
+        head = cls(magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id)
+        
+        if head.length != len(data):
+            raise ADBHeaderException(f"Length is incorrect! Expect {head.length}, got {len(data)}")
+        
+        return head
+    
+    def validate(self) -> bool:
+        if self.magic != 0xa13e:
+            raise ADBHeaderException(f"Magic {self.magic} != 0xa13e")
+        
+        if self.protocol_ver < 0x1000:
+            raise ADBHeaderException(f"Protocol version {hex(self.protocol_ver)} is invalid!")
+
+        if re.fullmatch(r"^S[0-9A-Z]{3}[P]?$", self.game_id) is None:
+            raise ADBHeaderException(f"Game ID {self.game_id} is invalid!")
+        
+        if self.store_id == 0:
+            raise ADBHeaderException(f"Store ID cannot be 0!")
+
+        if re.fullmatch(r"^A[0-9]{2}[E|X][0-9]{2}[A-HJ-NP-Z][0-9]{4}$", self.keychip_id) is None:
+            raise ADBHeaderException(f"Keychip ID {self.keychip_id} is invalid!")
+        
+        return True
+
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "magic" / Int16ul,
+            "unknown" / Int16ul,
+            "response_code" / Int16ul,
+            "length" / Int16ul,
+            "status" / Int16ul,
+            "game_id" / PaddedString(6, 'utf_8'),
+            "store_id" / Int32ul,
+            "keychip_id" / PaddedString(12, 'utf_8'),
+        )
+
+        return resp_struct.build(dict(
+            magic=self.magic,
+            unknown=self.protocol_ver,
+            response_code=self.cmd,
+            length=self.length,
+            status=self.status.value,
+            game_id = self.game_id,
+            store_id = self.store_id,
+            keychip_id = self.keychip_id,
+        ))
+
+class ADBBaseRequest:
+    def __init__(self, data: bytes) -> None:
+        self.head = ADBHeader.from_data(data)
+
+class ADBBaseResponse:
+    def __init__(self, code: int = 0, length: int = 0x20, status: int = 1, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 0x3087) -> None:
+        self.head = ADBHeader(0xa13e, protocol_ver, code, length, status, game_id, store_id, keychip_id)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, cmd: int, length: int = 0x20, status: int = 1) -> "ADBBaseResponse":
+        return cls(cmd, length, status, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+
+    def append_padding(self, data: bytes):
+        """Appends 0s to the end of the data until it's at the correct size"""
+        padding_size = self.head.length - len(data)
+        data += bytes(padding_size)
+        return data
+
+    def make(self) -> bytes:
+        return self.head.make()

core/adb_handlers/campaign.py

@@ -0,0 +1,132 @@
+from construct import Struct, Int16ul, Padding, Bytes, Int32ul, Int32sl
+
+from .base import *
+
+class Campaign:
+    def __init__(self) -> None:
+        self.id = 0
+        self.name = ""
+        self.announce_date = 0
+        self.start_date = 0
+        self.end_date = 0
+        self.distrib_start_date = 0
+        self.distrib_end_date = 0
+    
+    def make(self) -> bytes:
+        name_padding = bytes(128 - len(self.name))
+        return Struct(
+            "id" / Int32ul,
+            "name" / Bytes(128),
+            "announce_date" / Int32ul,
+            "start_date" / Int32ul,
+            "end_date" / Int32ul,
+            "distrib_start_date" / Int32ul,
+            "distrib_end_date" / Int32ul,
+            Padding(8),
+        ).build(dict(
+            id = self.id,
+            name = self.name.encode() + name_padding,
+            announce_date = self.announce_date,
+            start_date = self.start_date,
+            end_date = self.end_date,
+            distrib_start_date = self.distrib_start_date,
+            distrib_end_date = self.distrib_end_date,
+        ))
+
+class CampaignClear:
+    def __init__(self) -> None:
+        self.id = 0
+        self.entry_flag = 0
+        self.clear_flag = 0
+    
+    def make(self) -> bytes:
+        return Struct(
+            "id" / Int32ul,
+            "entry_flag" / Int32ul,
+            "clear_flag" / Int32ul,
+            Padding(4),
+        ).build(dict(
+            id = self.id,
+            entry_flag = self.entry_flag,
+            clear_flag = self.clear_flag,
+        ))
+
+class ADBCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x200, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaigns = [Campaign(), Campaign(), Campaign()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaigns:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBOldCampaignRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.campaign_id = struct.unpack_from("<I", data, 0x20)
+
+class ADBOldCampaignResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0C, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.info0 = 0
+        self.info1 = 0
+        self.info2 = 0
+        self.info3 = 0
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "info0" / Int32sl,
+            "info1" / Int32sl,
+            "info2" / Int32sl,
+            "info3" / Int32sl,
+        ).build(
+            info0 = self.info0,
+            info1 = self.info1,
+            info2 = self.info2,
+            info3 = self.info3,
+        )
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+        return self.head.make() + resp_struct
+
+class ADBCampaignClearRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id = struct.unpack_from("<i", data, 0x20)
+
+class ADBCampaignClearResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x0E, length: int = 0x50, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.campaign_clear_status = [CampaignClear(), CampaignClear(), CampaignClear()]
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBCampaignResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:
+        body = b""
+        
+        for c in self.campaign_clear_status:
+            body += c.make()
+        
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/felica.py

@@ -0,0 +1,85 @@
+from construct import Struct, Int32sl, Padding, Int8ub, Int16sl
+from typing import Union
+from .base import *
+
+class ADBFelicaLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        idm, pmm = struct.unpack_from(">QQ", data, 0x20)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+
+class ADBFelicaLookupResponse(ADBBaseResponse):
+    def __init__(self, access_code: str = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x03, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+    
+    @classmethod
+    def from_req(cls, req: ADBHeader, access_code: str = None) -> "ADBFelicaLookupResponse":
+        c = cls(access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+    
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "felica_idx" / Int32ul,
+            "access_code" / Int8ub[10],
+            Padding(2)
+        ).build(dict(
+            felica_idx = 0,
+            access_code = bytes.fromhex(self.access_code)
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct
+
+class ADBFelicaLookup2Request(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.random = struct.unpack_from("<16s", data, 0x20)[0]
+        idm, pmm = struct.unpack_from(">QQ", data, 0x30)
+        self.card_key_ver, self.write_ct, self.maca, company, fw_ver, self.dfc = struct.unpack_from("<16s16sQccH", data, 0x40)
+        self.idm = hex(idm)[2:].upper()
+        self.pmm = hex(pmm)[2:].upper()
+        self.company = CompanyCodes(int.from_bytes(company, 'little'))
+        self.fw_ver = ReaderFwVer.from_byte(fw_ver)
+
+class ADBFelicaLookup2Response(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None] = None, access_code: Union[str, None] = None, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x12, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.access_code = access_code if access_code is not None else "00000000000000000000"
+        self.company = CompanyCodes.SEGA
+        self.portal_status = PortalRegStatus.NO_REG
+        self.auth_key = [0] * 256
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None] = None, access_code: Union[str, None] = None) -> "ADBFelicaLookup2Response":
+        c = cls(user_id, access_code, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self) -> bytes:        
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+            "access_code" / Int8ub[10],
+            "portal_status" / Int8ub,
+            "company_code" / Int8ub,
+            Padding(8),
+            "auth_key" / Int8ub[256],
+        ).build(dict(
+            user_id = self.user_id,
+            relation1 = -1, # Unsupported
+            relation2 = -1, # Unsupported
+            access_code = bytes.fromhex(self.access_code),
+            portal_status = self.portal_status.value,
+            company_code = self.company.value,
+            auth_key = self.auth_key
+        ))
+
+        self.head.length = HEADER_SIZE + len(resp_struct)
+
+        return self.head.make() + resp_struct

core/adb_handlers/log.py

@@ -0,0 +1,56 @@
+from construct import Struct, Padding, Int8sl
+from typing import Final, List
+
+from .base import *
+NUM_LOGS: Final[int] = 20
+NUM_LEN_LOG_EX: Final[int] = 48
+
+class AmLogEx:
+    def __init__(self, data: bytes) -> None:
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct, self.local_time, \
+            self.tseq, self.place_id = struct.unpack("<IIQiii4xQiI", data)
+        self.status = LogStatus(status)
+
+class ADBStatusLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status = struct.unpack_from("<II", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.aime_id, status, self.user_id, self.credit_ct, self.bet_ct, self.won_ct = struct.unpack_from("<IIQiii", data, 0x20)
+        self.status = LogStatus(status)
+
+class ADBLogExRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.logs: List[AmLogEx] = []
+
+        for x in range(NUM_LOGS):
+            self.logs.append(AmLogEx(data[0x20 + (NUM_LEN_LOG_EX * x): 0x50 + (NUM_LEN_LOG_EX * x)]))
+        
+        self.num_logs = struct.unpack_from("<I", data, 0x03E0)[0]
+
+class ADBLogExResponse(ADBBaseResponse):
+    def __init__(self, game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", protocol_ver: int = 12423, code: int = 20, length: int = 64, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id, protocol_ver)
+
+    @classmethod
+    def from_req(cls, req: ADBHeader) -> "ADBLogExResponse":
+        c = cls(req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
+        return c
+    
+    def make(self) -> bytes:
+        resp_struct = Struct(
+            "log_result" / Int8sl[NUM_LOGS],
+            Padding(12)
+        )
+
+        body = resp_struct.build(dict(
+            log_result = [1] * NUM_LOGS
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/adb_handlers/lookup.py

@@ -0,0 +1,82 @@
+from construct import Struct, Int32sl, Padding, Int8sl
+from typing import Union
+
+from .base import *
+
+class ADBLookupException(Exception):
+    pass
+
+class ADBLookupRequest(ADBBaseRequest):
+    def __init__(self, data: bytes) -> None:
+        super().__init__(data)
+        self.access_code = data[0x20:0x2A].hex()
+        company_code, fw_version, self.serial_number = struct.unpack_from("<bbI", data, 0x2A)
+        
+        try:
+            self.company_code = CompanyCodes(company_code)
+        except ValueError as e:
+            raise ADBLookupException(f"Invalid company code - {e}")
+        
+        self.fw_version = ReaderFwVer.from_byte(fw_version)
+
+class ADBLookupResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", code: int = 0x06, length: int = 0x30, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(11)
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body
+
+class ADBLookupExResponse(ADBBaseResponse):
+    def __init__(self, user_id: Union[int, None], game_id: str = "SXXX", store_id: int = 1, keychip_id: str = "A69E01A8888", 
+                code: int = 0x10, length: int = 0x130, status: int = 1) -> None:
+        super().__init__(code, length, status, game_id, store_id, keychip_id)
+        self.user_id = user_id if user_id is not None else -1
+        self.portal_reg = PortalRegStatus.NO_REG
+        self.auth_key = [0] * 256
+
+    @classmethod
+    def from_req(cls, req: ADBHeader, user_id: Union[int, None]) -> "ADBLookupExResponse":
+        c = cls(user_id, req.game_id, req.store_id, req.keychip_id)
+        c.head.protocol_ver = req.protocol_ver
+        return c
+
+    def make(self):
+        resp_struct = Struct(
+            "user_id" / Int32sl,
+            "portal_reg" / Int8sl,
+            Padding(3),
+            "auth_key" / Int8sl[256],
+            "relation1" / Int32sl,
+            "relation2" / Int32sl,
+        )
+
+        body = resp_struct.build(dict(
+            user_id = self.user_id,
+            portal_reg = self.portal_reg.value,
+            auth_key = self.auth_key,
+            relation1 = -1,
+            relation2 = -1
+        ))
+
+        self.head.length = HEADER_SIZE + len(body)
+        return self.head.make() + body

core/aimedb.py

@@ -2,27 +2,18 @@ from twisted.internet.protocol import Factory, Protocol
 import logging, coloredlogs
 from Crypto.Cipher import AES
 import struct
-from typing import Dict, Any
+from typing import Dict, Tuple, Callable, Union
+from typing_extensions import Final
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
+from core.utils import create_sega_auth_key
 from core.data import Data
+from .adb_handlers import *
 
 
 class AimedbProtocol(Protocol):
-    AIMEDB_RESPONSE_CODES = {
-        "felica_lookup": 0x03,
-        "lookup": 0x06,
-        "log": 0x0A,
-        "campaign": 0x0C,
-        "touch": 0x0E,
-        "lookup2": 0x10,
-        "felica_lookup2": 0x12,
-        "log2": 0x14,
-        "hello": 0x65,
-    }
-
-    request_list: Dict[int, Any] = {}
+    request_list: Dict[int, Tuple[Callable[[bytes, int], Union[ADBBaseResponse, bytes]], int, str]] = {}
 
     def __init__(self, core_cfg: CoreConfig) -> None:
         self.logger = logging.getLogger("aimedb")
@@ -32,16 +23,27 @@ class AimedbProtocol(Protocol):
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
 
-        self.request_list[0x01] = self.handle_felica_lookup
-        self.request_list[0x04] = self.handle_lookup
-        self.request_list[0x05] = self.handle_register
-        self.request_list[0x09] = self.handle_log
-        self.request_list[0x0B] = self.handle_campaign
-        self.request_list[0x0D] = self.handle_touch
-        self.request_list[0x0F] = self.handle_lookup2
-        self.request_list[0x11] = self.handle_felica_lookup2
-        self.request_list[0x13] = self.handle_log2
-        self.request_list[0x64] = self.handle_hello
+        self.register_handler(0x01, 0x03, self.handle_felica_lookup, 'felica_lookup')
+        self.register_handler(0x02, 0x03, self.handle_felica_register, 'felica_register')
+
+        self.register_handler(0x04, 0x06, self.handle_lookup, 'lookup')
+        self.register_handler(0x05, 0x06, self.handle_register, 'register')
+
+        self.register_handler(0x07, 0x08, self.handle_status_log, 'status_log')
+        self.register_handler(0x09, 0x0A, self.handle_log, 'aime_log')       
+
+        self.register_handler(0x0B, 0x0C, self.handle_campaign, 'campaign')
+        self.register_handler(0x0D, 0x0E, self.handle_campaign_clear, 'campaign_clear')
+
+        self.register_handler(0x0F, 0x10, self.handle_lookup_ex, 'lookup_ex')
+        self.register_handler(0x11, 0x12, self.handle_felica_lookup_ex, 'felica_lookup_ex')
+
+        self.register_handler(0x13, 0x14, self.handle_log_ex, 'aime_log_ex')
+        self.register_handler(0x64, 0x65, self.handle_hello, 'hello')
+        self.register_handler(0x66, 0, self.handle_goodbye, 'goodbye')
+    
+    def register_handler(self, cmd: int, resp:int, handler: Callable[[bytes, int], Union[ADBBaseResponse, bytes]], name: str) -> None:
+        self.request_list[cmd] = (handler, resp, name)
 
     def append_padding(self, data: bytes):
         """Appends 0s to the end of the data until it's at the correct size"""
@@ -63,202 +65,269 @@ class AimedbProtocol(Protocol):
 
         try:
             decrypted = cipher.decrypt(data)
-        except:
-            self.logger.error(f"Failed to decrypt {data.hex()}")
+        
+        except Exception as e:
+            self.logger.error(f"Failed to decrypt {data.hex()} because {e}")
             return None
 
         self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
 
-        if not decrypted[1] == 0xA1 and not decrypted[0] == 0x3E:
-            self.logger.error(f"Bad magic")
-            return None
+        try:
+            head = ADBHeader.from_data(decrypted)
+        
+        except ADBHeaderException as e:
+            self.logger.error(f"Error parsing ADB header: {e}")
+            try:    
+                encrypted = cipher.encrypt(ADBBaseResponse().make())
+                self.transport.write(encrypted)
 
-        req_code = decrypted[4]
-
-        if req_code == 0x66:
-            self.logger.info(f"goodbye from {self.transport.getPeer().host}")
-            self.transport.loseConnection()
+            except Exception as e:
+                self.logger.error(f"Failed to encrypt default response because {e}")
+            
             return
 
-        try:
-            resp = self.request_list[req_code](decrypted)
-            encrypted = cipher.encrypt(resp)
-            self.logger.debug(f"Response {resp.hex()}")
+        if head.keychip_id == "ABCD1234567" or head.store_id == 0xfff0:
+            self.logger.warning(f"Request from uninitialized AMLib: {vars(head)}")
+
+        handler, resp_code, name = self.request_list.get(head.cmd, (self.handle_default, None, 'default'))
+
+        if resp_code is None:
+            self.logger.warning(f"No handler for cmd {hex(head.cmd)}")
+        
+        elif resp_code > 0:
+            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {self.transport.getPeer().host}")
+        
+        resp = handler(decrypted, resp_code)
+
+        if type(resp) == ADBBaseResponse or issubclass(type(resp), ADBBaseResponse):
+            resp_bytes = resp.make()
+            if len(resp_bytes) != resp.head.length:
+                resp_bytes = self.append_padding(resp_bytes)
+
+        elif type(resp) == bytes:
+            resp_bytes = resp
+        
+        elif resp is None: # Nothing to send, probably a goodbye
+            return
+        
+        else:
+            raise TypeError(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
+
+        try:    
+            encrypted = cipher.encrypt(resp_bytes)
+            self.logger.debug(f"Response {resp_bytes.hex()}")
             self.transport.write(encrypted)
 
-        except KeyError:
-            self.logger.error(f"Unknown command code {hex(req_code)}")
-            return None
+        except Exception as e:
+            self.logger.error(f"Failed to encrypt {resp_bytes.hex()} because {e}")
+        
+    def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
+        req = ADBHeader.from_data(data)
+        return ADBBaseResponse(resp_code, length, 1, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
 
-        except ValueError as e:
-            self.logger.error(f"Failed to encrypt {resp.hex()} because {e}")
-            return None
+    def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        return self.handle_default(data, resp_code)
 
-    def handle_campaign(self, data: bytes) -> bytes:
-        self.logger.info(f"campaign from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["campaign"],
-            0x0200,
-            0x0001,
-        )
-        return self.append_padding(ret)
+    def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        h = ADBHeader.from_data(data)
+        if h.protocol_ver >= 0x3030:
+            req = h
+            resp = ADBCampaignResponse.from_req(req)
 
-    def handle_hello(self, data: bytes) -> bytes:
-        self.logger.info(f"hello from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["hello"], 0x0020, 0x0001
-        )
-        return self.append_padding(ret)
-
-    def handle_lookup(self, data: bytes) -> bytes:
-        luid = data[0x20:0x2A].hex()
-        user_id = self.data.card.get_user_id_from_card(access_code=luid)
-
-        if user_id is None:
-            user_id = -1
-
-        self.logger.info(
-            f"lookup from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
-        )
-
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0130, 0x0001
-        )
-        ret += bytes(0x20 - len(ret))
-
-        if user_id is None:
-            ret += struct.pack("<iH", -1, 0)
         else:
-            ret += struct.pack("<l", user_id)
-        return self.append_padding(ret)
+            req = ADBOldCampaignRequest(data)
+            
+            self.logger.info(f"Legacy campaign request for campaign {req.campaign_id} (protocol version {hex(h.protocol_ver)})")
+            resp = ADBOldCampaignResponse.from_req(req.head)
+        
+        # We don't currently support campaigns
+        return resp
 
-    def handle_lookup2(self, data: bytes) -> bytes:
-        self.logger.info(f"lookup2")
+    def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBLookupRequest(data)
+        user_id = self.data.card.get_user_id_from_card(req.access_code)
+        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_locked = self.data.card.get_card_locked(req.access_code)
 
-        ret = bytearray(self.handle_lookup(data))
-        ret[4] = self.AIMEDB_RESPONSE_CODES["lookup2"]
-
-        return bytes(ret)
-
-    def handle_felica_lookup(self, data: bytes) -> bytes:
-        idm = data[0x20:0x28].hex()
-        pmm = data[0x28:0x30].hex()
-        access_code = self.data.card.to_access_code(idm)
+        if is_banned and is_locked:
+            ret.head.status = ADBStatus.BAN_SYS_USER
+        elif is_banned:
+            ret.head.status = ADBStatus.BAN_SYS
+        elif is_locked:
+            ret.head.status = ADBStatus.LOCK_USER
+        ret = ADBLookupResponse.from_req(req.head, user_id)
+        
         self.logger.info(
-            f"felica_lookup from {self.transport.getPeer().host}: idm {idm} pmm {pmm} -> access_code {access_code}"
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
+        )
+        return ret
+
+    def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBLookupRequest(data)
+        user_id = self.data.card.get_user_id_from_card(req.access_code)
+
+        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_locked = self.data.card.get_card_locked(req.access_code)
+
+        ret = ADBLookupExResponse.from_req(req.head, user_id)
+        if is_banned and is_locked:
+            ret.head.status = ADBStatus.BAN_SYS_USER
+        elif is_banned:
+            ret.head.status = ADBStatus.BAN_SYS
+        elif is_locked:
+            ret.head.status = ADBStatus.LOCK_USER
+
+        self.logger.info(
+            f"access_code {req.access_code} -> user_id {ret.user_id}"
         )
 
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["felica_lookup"],
-            0x0030,
-            0x0001,
+        if user_id and user_id > 0 and self.config.aimedb.id_secret:
+            auth_key = create_sega_auth_key(user_id, req.head.game_id, req.head.store_id, req.head.keychip_id, self.config.aimedb.id_secret, self.config.aimedb.id_lifetime_seconds)
+            if auth_key is not None:
+                auth_key_extra_len = 256 - len(auth_key)
+                auth_key_full = auth_key.encode() + (b"\0" * auth_key_extra_len)
+                self.logger.debug(f"Generated auth token {auth_key}")
+                ret.auth_key = auth_key_full
+
+        return ret
+
+    def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
+        """
+        On official, I think a card has to be registered for this to actually work, but 
+        I'm making the executive decision to not implement that and just kick back our
+        faux generated access code. The real felica IDm -> access code conversion is done
+        on the ADB server, which we do not and will not ever have access to. Because we can
+        assure that all IDms will be unique, this basic 0-padded hex -> int conversion will
+        be fine.
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
+        self.logger.info(
+            f"idm {req.idm} ipm {req.pmm} -> access_code {ac}"
         )
-        ret += bytes(26)
-        ret += bytes.fromhex(access_code)
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-        return self.append_padding(ret)
+    def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
+        """
+        I've never seen this used.
+        """
+        req = ADBFelicaLookupRequest(data)
+        ac = self.data.card.to_access_code(req.idm)
+        
+        if self.config.server.allow_user_registration:
+            user_id = self.data.user.create_user()
 
-    def handle_felica_lookup2(self, data: bytes) -> bytes:
-        idm = data[0x30:0x38].hex()
-        pmm = data[0x38:0x40].hex()
-        access_code = self.data.card.to_access_code(idm)
+            if user_id is None:
+                self.logger.error("Failed to register user!")
+                user_id = -1
+
+            else:
+                card_id = self.data.card.create_card(user_id, ac)
+
+                if card_id is None:
+                    self.logger.error("Failed to register card!")
+                    user_id = -1
+
+            self.logger.info(
+                f"Register access code {ac} (IDm: {req.idm} PMm: {req.pmm}) -> user_id {user_id}"
+            )
+
+        else:
+            self.logger.info(
+                f"Registration blocked!: access code {ac} (IDm: {req.idm} PMm: {req.pmm})"
+            )
+
+        return ADBFelicaLookupResponse.from_req(req.head, ac)
+
+    def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBFelicaLookup2Request(data)
+        access_code = self.data.card.to_access_code(req.idm)
         user_id = self.data.card.get_user_id_from_card(access_code=access_code)
 
         if user_id is None:
             user_id = -1
 
         self.logger.info(
-            f"felica_lookup2 from {self.transport.getPeer().host}: idm {idm} ipm {pmm} -> access_code {access_code} user_id {user_id}"
+            f"idm {req.idm} ipm {req.pmm} -> access_code {access_code} user_id {user_id}"
         )
 
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["felica_lookup2"],
-            0x0140,
-            0x0001,
-        )
-        ret += bytes(22)
-        ret += struct.pack("<lq", user_id, -1)  # first -1 is ext_id, 3rd is access code
-        ret += bytes.fromhex(access_code)
-        ret += struct.pack("<l", 1)
+        resp = ADBFelicaLookup2Response.from_req(req.head, user_id, access_code)
 
-        return self.append_padding(ret)
+        if user_id and user_id > 0 and self.config.aimedb.id_secret:
+            auth_key = create_sega_auth_key(user_id, req.head.game_id, req.head.store_id, req.head.keychip_id, self.config.aimedb.id_secret, self.config.aimedb.id_lifetime_seconds)
+            if auth_key is not None:
+                auth_key_extra_len = 256 - len(auth_key)
+                auth_key_full = auth_key.encode() + (b"\0" * auth_key_extra_len)
+                self.logger.debug(f"Generated auth token {auth_key}")
+                resp.auth_key = auth_key_full
 
-    def handle_touch(self, data: bytes) -> bytes:
-        self.logger.info(f"touch from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["touch"], 0x0050, 0x0001
-        )
-        ret += bytes(5)
-        ret += struct.pack("<3H", 0x6F, 0, 1)
+        return resp
 
-        return self.append_padding(ret)
+    def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+        req = ADBCampaignClearRequest(data)
+
+        resp = ADBCampaignClearResponse.from_req(req.head)
+
+        # We don't support campaign stuff
+        return resp
+
+    def handle_register(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLookupRequest(data)
+        user_id = -1
 
-    def handle_register(self, data: bytes) -> bytes:
-        luid = data[0x20:0x2A].hex()
         if self.config.server.allow_user_registration:
             user_id = self.data.user.create_user()
 
             if user_id is None:
-                user_id = -1
                 self.logger.error("Failed to register user!")
+                user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, luid)
+                card_id = self.data.card.create_card(user_id, req.access_code)
 
                 if card_id is None:
-                    user_id = -1
                     self.logger.error("Failed to register card!")
+                    user_id = -1
 
             self.logger.info(
-                f"register from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
+                f"Register access code {req.access_code} -> user_id {user_id}"
             )
 
         else:
             self.logger.info(
-                f"register from {self.transport.getPeer().host} blocked!: luid {luid}"
+                f"Registration blocked!: access code {req.access_code}"
             )
-            user_id = -1
 
-        ret = struct.pack(
-            "<5H",
-            0xA13E,
-            0x3087,
-            self.AIMEDB_RESPONSE_CODES["lookup"],
-            0x0030,
-            0x0001 if user_id > -1 else 0,
-        )
-        ret += bytes(0x20 - len(ret))
-        ret += struct.pack("<l", user_id)
+        resp = ADBLookupResponse.from_req(req.head, user_id)
+        if resp.user_id <= 0:
+            resp.head.status = ADBStatus.BAN_SYS # Closest we can get to a "You cannot register"
 
-        return self.append_padding(ret)
+        return resp
 
-    def handle_log(self, data: bytes) -> bytes:
-        # TODO: Save aimedb logs
-        self.logger.info(f"log from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log"], 0x0020, 0x0001
-        )
-        return self.append_padding(ret)
+    # TODO: Save these in some capacity, as deemed relevant
+    def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBStatusLogRequest(data)
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event")
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-    def handle_log2(self, data: bytes) -> bytes:
-        self.logger.info(f"log2 from {self.transport.getPeer().host}")
-        ret = struct.pack(
-            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log2"], 0x0040, 0x0001
-        )
-        ret += bytes(22)
-        ret += struct.pack("H", 1)
+    def handle_log(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLogRequest(data)
+        self.logger.info(f"User {req.aime_id} logged {req.status.name} event, credit_ct: {req.credit_ct} bet_ct: {req.bet_ct} won_ct: {req.won_ct}")
+        return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-        return self.append_padding(ret)
+    def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
+        req = ADBLogExRequest(data)
+        strs = []
+        self.logger.info(f"Recieved {req.num_logs} or {len(req.logs)} logs")
+        
+        for x in range(req.num_logs):
+            self.logger.debug(f"User {req.logs[x].aime_id} logged {req.logs[x].status.name} event, credit_ct: {req.logs[x].credit_ct} bet_ct: {req.logs[x].bet_ct} won_ct: {req.logs[x].won_ct}")
+        return ADBLogExResponse.from_req(req.head)
 
+    def handle_goodbye(self, data: bytes, resp_code: int) -> None:
+        self.logger.info(f"goodbye from {self.transport.getPeer().host}")
+        self.transport.loseConnection()
+        return
 
 class AimedbFactory(Factory):
     protocol = AimedbProtocol

core/allnet.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Any, Optional, Tuple
+from typing import Dict, List, Any, Optional, Tuple, Union, Final
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 from twisted.web.http import Request
@@ -6,17 +6,88 @@ from datetime import datetime
 import pytz
 import base64
 import zlib
+import json
+from enum import Enum
 from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
 from time import strptime
 from os import path
+import urllib.parse
+import math
 
-from core.config import CoreConfig
-from core.utils import Utils
-from core.data import Data
-from core.const import *
+from .config import CoreConfig
+from .utils import Utils
+from .data import Data
+from .const import *
+from .title import TitleServlet
 
+BILLING_DT_FORMAT: Final[str] = "%Y%m%d%H%M%S"
+
+class DLIMG_TYPE(Enum):
+    app = 0
+    opt = 1
+
+class ALLNET_STAT(Enum):
+    ok = 0
+    bad_game = -1
+    bad_machine = -2
+    bad_shop = -3
+
+class DLI_STATUS(Enum):
+    START = 0
+    GET_DOWNLOAD_CONFIGURATION = 1
+    WAIT_DOWNLOAD = 2
+    DOWNLOADING = 3
+
+    NOT_SPECIFY_DLI = 100
+    ONLY_POST_REPORT = 101
+    STOPPED_BY_APP_RELEASE = 102
+    STOPPED_BY_OPT_RELEASE = 103
+
+    DOWNLOAD_COMPLETE_RECENTLY = 110
+
+    DOWNLOAD_COMPLETE_WAIT_RELEASE_TIME = 120
+    DOWNLOAD_COMPLETE_BUT_NOT_SYNC_SERVER = 121
+    DOWNLOAD_COMPLETE_BUT_NOT_FIRST_RESUME = 122
+    DOWNLOAD_COMPLETE_BUT_NOT_FIRST_LAUNCH = 123
+    DOWNLOAD_COMPLETE_WAIT_UPDATE = 124
+
+    DOWNLOAD_COMPLETE_AND_ALREADY_UPDATE = 130
+
+    ERROR_AUTH_FAILURE = 200
+
+    ERROR_GET_DLI_HTTP = 300
+    ERROR_GET_DLI = 301
+    ERROR_PARSE_DLI = 302
+    ERROR_INVALID_GAME_ID = 303
+    ERROR_INVALID_IMAGE_LIST = 304
+    ERROR_GET_DLI_APP = 305
+
+    ERROR_GET_BOOT_ID = 400
+    ERROR_ACCESS_SERVER = 401
+    ERROR_NO_IMAGE = 402
+    ERROR_ACCESS_IMAGE = 403
+
+    ERROR_DOWNLOAD_APP = 500
+    ERROR_DOWNLOAD_OPT = 501
+
+    ERROR_DISK_FULL = 600
+    ERROR_UNINSTALL = 601
+    ERROR_INSTALL_APP = 602
+    ERROR_INSTALL_OPT = 603
+
+    ERROR_GET_DLI_INTERNAL = 900
+    ERROR_ICF = 901
+    ERROR_CHECK_RELEASE_INTERNAL = 902
+    UNKNOWN = 999 # Not the actual enum val but it needs to be here as a catch-all
+
+    @classmethod
+    def from_int(cls, num: int) -> "DLI_STATUS":
+        try:
+            return cls(num)
+        except ValueError:
+            return cls.UNKNOWN
 
 class AllnetServlet:
     def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
@@ -24,7 +95,6 @@ class AllnetServlet:
         self.config = core_cfg
         self.config_folder = cfg_folder
         self.data = Data(core_cfg)
-        self.uri_registry: Dict[str, Tuple[str, str]] = {}
 
         self.logger = logging.getLogger("allnet")
         if not hasattr(self.logger, "initialized"):
@@ -55,31 +125,29 @@ class AllnetServlet:
         if len(plugins) == 0:
             self.logger.error("No games detected!")
 
-        for _, mod in plugins.items():
-            if hasattr(mod, "index") and hasattr(mod.index, "get_allnet_info"):
-                for code in mod.game_codes:
-                    enabled, uri, host = mod.index.get_allnet_info(
-                        code, self.config, self.config_folder
-                    )
-
-                    if enabled:
-                        self.uri_registry[code] = (uri, host)
-
         self.logger.info(
-            f"Serving {len(self.uri_registry)} game codes port {core_cfg.allnet.port}"
+            f"Serving {len(TitleServlet.title_registry)} game codes port {core_cfg.allnet.port}"
         )
 
     def handle_poweron(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
+        pragma_header = request.getHeader('Pragma')
+        is_dfi = pragma_header is not None and pragma_header == "DFI"
+        
         try:
-            req_dict = self.allnet_req_to_dict(request.content.getvalue())
+            if is_dfi:
+                req_urlencode = self.from_dfi(request.content.getvalue())
+            else:
+                req_urlencode = request.content.getvalue().decode()
+
+            req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
                 raise AllnetRequestException()
 
             req = AllnetPowerOnRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
-            if not req.game_id or not req.ver or not req.serial or not req.ip:
+            if not req.game_id or not req.ver or not req.serial or not req.ip or not req.firm_ver or not req.boot_ver:
                 raise AllnetRequestException(
                     f"Bad auth request params from {request_ip} - {vars(req)}"
                 )
@@ -89,46 +157,53 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        if req.format_ver == "3":
+        if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
-        else:
+        elif req.format_ver == 2:
             resp = AllnetPowerOnResponse2()
+        else:
+            resp = AllnetPowerOnResponse()
 
         self.logger.debug(f"Allnet request: {vars(req)}")
-        if req.game_id not in self.uri_registry:
-            if not self.config.server.is_develop:
-                msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
-                self.data.base.log_event(
-                    "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
-                )
-                self.logger.warn(msg)
 
-                resp.stat = 0
-                return self.dict_to_http_form_string([vars(resp)])
-
-            else:
-                self.logger.info(
-                    f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
-                )
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
-                resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
-                return self.dict_to_http_form_string([vars(resp)])
-
-        resp.uri, resp.host = self.uri_registry[req.game_id]
-
-        machine = self.data.arcade.get_machine(req.serial)
+        machine = self.data.arcade.get_machine(req.serial)        
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {req.serial} attempted allnet auth from {request_ip}."
             self.data.base.log_event(
                 "allnet", "ALLNET_AUTH_UNKNOWN_SERIAL", logging.WARN, msg
             )
-            self.logger.warn(msg)
+            self.logger.warning(msg)
 
-            resp.stat = 0
-            return self.dict_to_http_form_string([vars(resp)])
+            resp.stat = ALLNET_STAT.bad_machine.value
+            resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
         if machine is not None:
             arcade = self.data.arcade.get_arcade(machine["arcade"])
+            if self.config.server.check_arcade_ip:
+                if arcade["ip"] and arcade["ip"] is not None and arcade["ip"] != req.ip:
+                    msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip} (expected {arcade['ip']})."
+                    self.data.base.log_event(
+                        "allnet", "ALLNET_AUTH_BAD_IP", logging.ERROR, msg
+                    )
+                    self.logger.warning(msg)
+
+                    resp.stat = ALLNET_STAT.bad_shop.value
+                    resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                
+                elif (not arcade["ip"] or arcade["ip"] is None) and self.config.server.strict_ip_checking:
+                    msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip}, but arcade {arcade['id']} has no IP set! (strict checking enabled)."
+                    self.data.base.log_event(
+                        "allnet", "ALLNET_AUTH_NO_SHOP_IP", logging.ERROR, msg
+                    )
+                    self.logger.warning(msg)
+
+                    resp.stat = ALLNET_STAT.bad_shop.value
+                    resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+
+
             country = (
                 arcade["country"] if machine["country"] is None else machine["country"]
             )
@@ -146,35 +221,76 @@ class AllnetServlet:
                 else AllnetJapanRegionId.AICHI.value
             )
             resp.region_name0 = (
-                arcade["country"]
-                if arcade["country"] is not None
-                else AllnetCountryCode.JAPAN.value
-            )
-            resp.region_name1 = (
                 arcade["state"]
                 if arcade["state"] is not None
                 else AllnetJapanRegionId.AICHI.name
             )
-            resp.region_name2 = arcade["city"] if arcade["city"] is not None else ""
-            resp.client_timezone = (
-                arcade["timezone"] if arcade["timezone"] is not None else "+0900"
+            resp.region_name1 = (
+                arcade["country"]
+                if arcade["country"] is not None
+                else AllnetCountryCode.JAPAN.value
             )
+            resp.region_name2 = arcade["city"] if arcade["city"] is not None else ""
+            resp.client_timezone = ( # lmao
+                arcade["timezone"] if arcade["timezone"] is not None else "+0900" if req.format_ver == 3 else "+09:00"
+            )
+        
+        if req.game_id not in TitleServlet.title_registry:
+            if not self.config.server.is_develop:
+                msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
+                self.data.base.log_event(
+                    "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
+                )
+                self.logger.warning(msg)
 
+                resp.stat = ALLNET_STAT.bad_game.value
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+
+            else:
+                self.logger.info(
+                    f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
+                )
+                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
+                resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
+                
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+                
+                self.logger.debug(f"Allnet response: {resp_str}")
+                return (resp_str + "\n").encode("utf-8")
+
+        
         int_ver = req.ver.replace(".", "")
-        resp.uri = resp.uri.replace("$v", int_ver)
-        resp.host = resp.host.replace("$v", int_ver)
+        resp.uri, resp.host = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial)
 
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
         self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
-        self.logger.debug(f"Allnet response: {vars(resp)}")
 
-        return self.dict_to_http_form_string([vars(resp)]).encode("utf-8")
+        resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+        self.logger.debug(f"Allnet response: {resp_dict}")        
+        resp_str += "\n"
+
+        """if is_dfi:
+            request.responseHeaders.addRawHeader('Pragma', 'DFI')
+            return self.to_dfi(resp_str)"""
+
+        return resp_str.encode("utf-8")
 
     def handle_dlorder(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
+        pragma_header = request.getHeader('Pragma')
+        is_dfi = pragma_header is not None and pragma_header == "DFI"
+        
         try:
-            req_dict = self.allnet_req_to_dict(request.content.getvalue())
+            if is_dfi:
+                req_urlencode = self.from_dfi(request.content.getvalue())
+            else:
+                req_urlencode = request.content.getvalue().decode()
+
+            req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
                 raise AllnetRequestException()
 
@@ -194,27 +310,34 @@ class AllnetServlet:
         self.logger.info(
             f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
         )
-        resp = AllnetDownloadOrderResponse()
+        resp = AllnetDownloadOrderResponse(serial=req.serial)
 
         if (
             not self.config.allnet.allow_online_updates
             or not self.config.allnet.update_cfg_folder
         ):
-            return self.dict_to_http_form_string([vars(resp)])
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
 
         else:  # TODO: Keychip check
             if path.exists(
-                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver}-app.ini"
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
             ):
                 resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
 
             if path.exists(
-                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver}-opt.ini"
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
             ):
                 resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
 
             self.logger.debug(f"Sending download uri {resp.uri}")
-            return self.dict_to_http_form_string([vars(resp)])
+            self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
+
+            res_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
+            """if is_dfi:
+                request.responseHeaders.addRawHeader('Pragma', 'DFI')
+                return self.to_dfi(res_str)"""
+
+            return res_str
 
     def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
         if "file" not in match:
@@ -223,6 +346,9 @@ class AllnetServlet:
         req_file = match["file"].replace("%0A", "")
 
         if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
+            self.logger.info(f"Request for DL INI file {req_file} from {Utils.get_ip_addr(request)} successful")
+            self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
+            
             return open(
                 f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
             ).read()
@@ -231,14 +357,71 @@ class AllnetServlet:
         return b""
 
     def handle_dlorder_report(self, request: Request, match: Dict) -> bytes:
-        self.logger.info(
-            f"DLI Report from {Utils.get_ip_addr(request)}: {request.content.getvalue()}"
-        )
-        return b""
+        req_raw = request.content.getvalue()
+        client_ip = Utils.get_ip_addr(request)
+        try:
+            req_dict: Dict = json.loads(req_raw)
+        except Exception as e:
+            self.logger.warning(f"Failed to parse DL Report: {e}")
+            return "NG"
+        
+        dl_data_type = DLIMG_TYPE.app
+        dl_data = req_dict.get("appimage", {})
+        
+        if dl_data is None or not dl_data:
+            dl_data_type = DLIMG_TYPE.opt
+            dl_data = req_dict.get("optimage", {})
+        
+        if dl_data is None or not dl_data:
+            self.logger.warning(f"Failed to parse DL Report: Invalid format - contains neither appimage nor optimage")
+            return "NG"
+
+        rep = DLReport(dl_data, dl_data_type)
+
+        if not rep.validate():
+            self.logger.warning(f"Failed to parse DL Report: Invalid format - {rep.err}")
+            return "NG"
+        
+        msg = f"{rep.serial} @ {client_ip} reported {rep.rep_type.name} download state {rep.rf_state.name} for {rep.gd} v{rep.dav}:"\
+              f" {rep.tdsc}/{rep.tsc} segments downloaded for working files {rep.wfl} with {rep.dfl if rep.dfl else 'none'} complete."
+        
+        self.data.base.log_event("allnet", "DL_REPORT", logging.INFO, msg, dl_data)
+        self.logger.info(msg)
+
+        return "OK"
+
+    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
+        req_data = request.content.getvalue()
+        sections = req_data.decode("utf-8").split("\r\n")
+        
+        req_dict = dict(urllib.parse.parse_qsl(sections[0]))
+
+        serial: Union[str, None] = req_dict.get("serial", None)
+        num_files_to_dl: Union[str, None] = req_dict.get("nb_ftd", None)
+        num_files_dld: Union[str, None] = req_dict.get("nb_dld", None)
+        dl_state: Union[str, None] = req_dict.get("dld_st", None)
+        ip = Utils.get_ip_addr(request)
+
+        if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
+            return "NG".encode()
+
+        self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
+        return "OK".encode()
+    
+    def handle_alive(self, request: Request, match: Dict) -> bytes:
+        return "OK".encode()
 
     def handle_billing_request(self, request: Request, _: Dict):
-        req_dict = self.billing_req_to_dict(request.content.getvalue())
+        req_raw = request.content.getvalue()
+        
+        if request.getHeader('Content-Type') == "application/octet-stream":
+            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
+        else:
+            req_unzip = req_raw
+        
+        req_dict = self.billing_req_to_dict(req_unzip)
         request_ip = Utils.get_ip_addr(request)
+        
         if req_dict is None:
             self.logger.error(f"Failed to parse request {request.content.getvalue()}")
             return b""
@@ -248,41 +431,61 @@ class AllnetServlet:
         rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
         signer = PKCS1_v1_5.new(rsa)
         digest = SHA.new()
+        traces: List[TraceData] = []
+        try:
+            req = BillingInfo(req_dict[0])
+        except KeyError as e:
+            self.logger.error(f"Billing request failed to parse: {e}")
+            return f"result=5&linelimit=&message=field is missing or formatting is incorrect\r\n".encode()
 
-        kc_playlimit = int(req_dict[0]["playlimit"])
-        kc_nearfull = int(req_dict[0]["nearfull"])
-        kc_billigtype = int(req_dict[0]["billingtype"])
-        kc_playcount = int(req_dict[0]["playcnt"])
-        kc_serial: str = req_dict[0]["keychipid"]
-        kc_game: str = req_dict[0]["gameid"]
-        kc_date = strptime(req_dict[0]["date"], "%Y%m%d%H%M%S")
-        kc_serial_bytes = kc_serial.encode()
+        for x in range(1, len(req_dict)):
+            if not req_dict[x]:
+                continue
+            
+            try:
+                tmp = TraceData(req_dict[x])
+                if tmp.trace_type == TraceDataType.CHARGE:
+                    tmp = TraceDataCharge(req_dict[x])
+                elif tmp.trace_type == TraceDataType.EVENT:
+                    tmp = TraceDataEvent(req_dict[x])
+                elif tmp.trace_type == TraceDataType.CREDIT:
+                    tmp = TraceDataCredit(req_dict[x])
+                
+                traces.append(tmp)
+            
+            except KeyError as e:
+                self.logger.warn(f"Tracelog failed to parse: {e}")
 
-        machine = self.data.arcade.get_machine(kc_serial)
+        kc_serial_bytes = req.keychipid.encode()
+        
+
+        machine = self.data.arcade.get_machine(req.keychipid)
         if machine is None and not self.config.server.allow_unregistered_serials:
-            msg = f"Unrecognised serial {kc_serial} attempted billing checkin from {request_ip} for game {kc_game}."
+            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
             self.data.base.log_event(
                 "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
             )
-            self.logger.warn(msg)
+            self.logger.warning(msg)
 
-            resp = BillingResponse("", "", "", "")
-            resp.result = "1"
-            return self.dict_to_http_form_string([vars(resp)])
+            return f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n".encode()
 
         msg = (
-            f"Billing checkin from {request_ip}: game {kc_game} keychip {kc_serial} playcount "
-            f"{kc_playcount} billing_type {kc_billigtype} nearfull {kc_nearfull} playlimit {kc_playlimit}"
+            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
+            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
         )
         self.logger.info(msg)
         self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
+        if req.traceleft > 0:
+            self.logger.warn(f"{req.traceleft} unsent tracelogs")
+        kc_playlimit = req.playlimit
+        kc_nearfull = req.nearfull
 
-        while kc_playcount > kc_playlimit:
+        while req.playcnt > req.playlimit:
             kc_playlimit += 1024
             kc_nearfull += 1024
 
         playlimit = kc_playlimit
-        nearfull = kc_nearfull + (kc_billigtype * 0x00010000)
+        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
 
         digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
         playlimit_sig = signer.sign(digest).hex()
@@ -293,44 +496,33 @@ class AllnetServlet:
 
         # TODO: playhistory
 
-        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
+        #resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
+        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
 
-        resp_str = self.dict_to_http_form_string([vars(resp)], True)
-        if resp_str is None:
-            self.logger.error(f"Failed to parse response {vars(resp)}")
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
 
         self.logger.debug(f"response {vars(resp)}")
+        if req.traceleft > 0:
+            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
+            return f"result=6&waittime=0&linelimit=20\r\n".encode()
+
         return resp_str.encode("utf-8")
 
     def handle_naomitest(self, request: Request, _: Dict) -> bytes:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
         return b"naomi ok"
 
-    def kvp_to_dict(self, kvp: List[str]) -> List[Dict[str, Any]]:
-        ret: List[Dict[str, Any]] = []
-        for x in kvp:
-            items = x.split("&")
-            tmp = {}
-
-            for item in items:
-                kvp = item.split("=")
-                if len(kvp) == 2:
-                    tmp[kvp[0]] = kvp[1]
-
-            ret.append(tmp)
-
-        return ret
-
     def billing_req_to_dict(self, data: bytes):
         """
         Parses an billing request string into a python dictionary
         """
         try:
-            decomp = zlib.decompressobj(-zlib.MAX_WBITS)
-            unzipped = decomp.decompress(data)
-            sections = unzipped.decode("ascii").split("\r\n")
+            sections = data.decode("ascii").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
@@ -341,111 +533,94 @@ class AllnetServlet:
         Parses an allnet request string into a python dictionary
         """
         try:
-            zipped = base64.b64decode(data)
-            unzipped = zlib.decompress(zipped)
-            sections = unzipped.decode("utf-8").split("\r\n")
+            sections = data.split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"allnet_req_to_dict: {e} while parsing {data}")
             return None
 
-    def dict_to_http_form_string(
-        self,
-        data: List[Dict[str, Any]],
-        crlf: bool = False,
-        trailing_newline: bool = True,
-    ) -> Optional[str]:
-        """
-        Takes a python dictionary and parses it into an allnet response string
-        """
-        try:
-            urlencode = ""
-            for item in data:
-                for k, v in item.items():
-                    urlencode += f"{k}={v}&"
+    def from_dfi(self, data: bytes) -> str:
+        zipped = base64.b64decode(data)
+        unzipped = zlib.decompress(zipped)
+        return unzipped.decode("utf-8")
 
-                if crlf:
-                    urlencode = urlencode[:-1] + "\r\n"
-                else:
-                    urlencode = urlencode[:-1] + "\n"
-
-            if not trailing_newline:
-                if crlf:
-                    urlencode = urlencode[:-2]
-                else:
-                    urlencode = urlencode[:-1]
-
-            return urlencode
-
-        except Exception as e:
-            self.logger.error(f"dict_to_http_form_string: {e} while parsing {data}")
-            return None
+    def to_dfi(self, data: str) -> bytes:
+        unzipped = data.encode('utf-8')
+        zipped = zlib.compress(unzipped)
+        return base64.b64encode(zipped)
 
 
 class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
         if req is None:
             raise AllnetRequestException("Request processing failed")
-        self.game_id: str = req.get("game_id", "")
-        self.ver: str = req.get("ver", "")
-        self.serial: str = req.get("serial", "")
-        self.ip: str = req.get("ip", "")
-        self.firm_ver: str = req.get("firm_ver", "")
-        self.boot_ver: str = req.get("boot_ver", "")
-        self.encode: str = req.get("encode", "")
-        self.hops = int(req.get("hops", "0"))
-        self.format_ver = req.get("format_ver", "2")
-        self.token = int(req.get("token", "0"))
+        self.game_id: str = req.get("game_id", None)
+        self.ver: str = req.get("ver", None)
+        self.serial: str = req.get("serial", None)
+        self.ip: str = req.get("ip", None)
+        self.firm_ver: str = req.get("firm_ver", None)
+        self.boot_ver: str = req.get("boot_ver", None)
+        self.encode: str = req.get("encode", "EUC-JP")
+        self.hops = int(req.get("hops", "-1"))
+        self.format_ver = float(req.get("format_ver", "1.00"))
+        self.token: str = req.get("token", "0")
 
-
-class AllnetPowerOnResponse3:
-    def __init__(self, token) -> None:
-        self.stat = 1
-        self.uri = ""
-        self.host = ""
-        self.place_id = "123"
-        self.name = ""
-        self.nickname = ""
-        self.region0 = "1"
-        self.region_name0 = "W"
-        self.region_name1 = ""
-        self.region_name2 = ""
-        self.region_name3 = ""
-        self.country = "JPN"
-        self.allnet_id = "123"
-        self.client_timezone = "+0900"
-        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
-            "%Y-%m-%dT%H:%M:%SZ"
-        )
-        self.setting = "1"
-        self.res_ver = "3"
-        self.token = str(token)
-
-
-class AllnetPowerOnResponse2:
+class AllnetPowerOnResponse:
     def __init__(self) -> None:
         self.stat = 1
         self.uri = ""
         self.host = ""
-        self.place_id = "123"
-        self.name = "Test"
-        self.nickname = "Test123"
+        self.place_id = "0123"
+        self.name = "ARTEMiS"
+        self.nickname = "ARTEMiS"
         self.region0 = "1"
         self.region_name0 = "W"
-        self.region_name1 = "X"
-        self.region_name2 = "Y"
-        self.region_name3 = "Z"
-        self.country = "JPN"
+        self.region_name1 = ""
+        self.region_name2 = ""
+        self.region_name3 = ""        
+        self.setting = "1"
         self.year = datetime.now().year
         self.month = datetime.now().month
         self.day = datetime.now().day
         self.hour = datetime.now().hour
         self.minute = datetime.now().minute
         self.second = datetime.now().second
-        self.setting = "1"
-        self.timezone = "+0900"
+
+class AllnetPowerOnResponse3(AllnetPowerOnResponse):
+    def __init__(self, token) -> None:
+        super().__init__()
+
+        # Added in v3
+        self.country = "JPN"
+        self.allnet_id = "123"
+        self.client_timezone = "+0900"
+        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
+            "%Y-%m-%dT%H:%M:%SZ"
+        )
+        self.res_ver = "3"
+        self.token = token
+
+        # Removed in v3
+        self.year = None
+        self.month = None
+        self.day = None
+        self.hour = None
+        self.minute = None
+        self.second = None
+
+
+class AllnetPowerOnResponse2(AllnetPowerOnResponse):
+    def __init__(self) -> None:
+        super().__init__()
+
+        # Added in v2
+        self.country = "JPN"
+        self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
 
@@ -463,6 +638,122 @@ class AllnetDownloadOrderResponse:
         self.serial = serial
         self.uri = uri
 
+class TraceDataType(Enum):
+    CHARGE = 0
+    EVENT = 1
+    CREDIT = 2
+
+class BillingType(Enum):
+    A = 1
+    B = 0
+
+class float5:
+    def __init__(self, n: str = "0") -> None:
+        nf = float(n)
+        if nf > 999.9 or nf < 0:
+            raise ValueError('float5 must be between 0.000 and 999.9 inclusive')
+        
+        return nf
+    
+    @classmethod
+    def to_str(cls, f: float):
+        return f"%.{2 - int(math.log10(f))+1}f" % f
+
+class BillingInfo:
+    def __init__(self, data: Dict) -> None:
+        try:
+            self.keychipid = str(data.get("keychipid", None))
+            self.functype = int(data.get("functype", None))
+            self.gameid = str(data.get("gameid", None))
+            self.gamever = float(data.get("gamever", None))
+            self.boardid = str(data.get("boardid", None))
+            self.tenpoip = str(data.get("tenpoip", None))
+            self.libalibver = float(data.get("libalibver", None))
+            self.datamax = int(data.get("datamax", None))
+            self.billingtype = BillingType(int(data.get("billingtype", None)))
+            self.protocolver = float(data.get("protocolver", None))
+            self.operatingfix = bool(data.get("operatingfix", None))
+            self.traceleft = int(data.get("traceleft", None))
+            self.requestno = int(data.get("requestno", None))
+            self.datesync = bool(data.get("datesync", None))
+            self.timezone = str(data.get("timezone", None))
+            self.date = datetime.strptime(data.get("date", None), BILLING_DT_FORMAT)
+            self.crcerrcnt = int(data.get("crcerrcnt", None))
+            self.memrepair = bool(data.get("memrepair", None))
+            self.playcnt = int(data.get("playcnt", None))
+            self.playlimit = int(data.get("playlimit", None))
+            self.nearfull = int(data.get("nearfull", None))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceData:
+    def __init__(self, data: Dict) -> None:
+        try:
+            self.crc_err_flg = bool(data.get("cs", None))
+            self.record_number = int(data.get("rn", None))
+            self.seq_number = int(data.get("sn", None))
+            self.trace_type = TraceDataType(int(data.get("tt", None)))
+            self.date_sync_flg = bool(data.get("ds", None))
+            
+            dt = data.get("dt", None)
+            if dt is None:
+                raise KeyError("dt not present")
+            if dt == "20000000000000": # Not sure what causes it to send like this...
+                self.date = datetime(2000, 1, 1, 0, 0, 0, 0)
+            else:
+                self.date = datetime.strptime(data.get("dt", None), BILLING_DT_FORMAT)
+            
+            self.keychip = str(data.get("kn", None))
+            self.lib_ver = float(data.get("alib", 0))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceDataCharge(TraceData):
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        try:
+            self.game_id = str(data.get("gi", None)) # these seem optional...?
+            self.game_version = float(data.get("gv", 0))
+            self.board_serial = str(data.get("bn", None))
+            self.shop_ip = str(data.get("ti", None))
+            self.play_count = int(data.get("pc", None))
+            self.play_limit = int(data.get("pl", None))
+            self.product_code = int(data.get("ic", None))
+            self.product_count = int(data.get("in", None))
+            self.func_type = int(data.get("kk", None))
+            self.player_number = int(data.get("playerno", None))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceDataEvent(TraceData):
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        try:
+            self.message = str(data.get("me", None))
+        except Exception as e:
+            raise KeyError(e)
+
+class TraceDataCredit(TraceData):
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        try:
+            self.chute_type = int(data.get("cct", None))
+            self.service_type = int(data.get("cst", None))
+            self.operation_type = int(data.get("cop", None))
+            self.coin_rate0 = int(data.get("cr0", None))
+            self.coin_rate1 = int(data.get("cr1", None))
+            self.bonus_addition = int(data.get("cba", None))
+            self.credit_rate = int(data.get("ccr", None))
+            self.credit0 = int(data.get("cc0", None))
+            self.credit1 = int(data.get("cc1", None))
+            self.credit2 = int(data.get("cc2", None))
+            self.credit3 = int(data.get("cc3", None))
+            self.credit4 = int(data.get("cc4", None))
+            self.credit5 = int(data.get("cc5", None))
+            self.credit6 = int(data.get("cc6", None))
+            self.credit7 = int(data.get("cc7", None))
+        except Exception as e:
+            raise KeyError(e)
 
 class BillingResponse:
     def __init__(
@@ -471,20 +762,22 @@ class BillingResponse:
         playlimit_sig: str = "",
         nearfull: str = "",
         nearfull_sig: str = "",
+        request_num: int = 1,
+        protocol_ver: float = 1.000,
         playhistory: str = "000000/0:000000/0:000000/0",
     ) -> None:
-        self.result = "0"
-        self.waitime = "100"
-        self.linelimit = "1"
-        self.message = ""
+        self.result = 0
+        self.requestno = request_num
+        self.traceerase = 1
+        self.fixinterval = 120
+        self.fixlogcnt = 100
         self.playlimit = playlimit
         self.playlimitsig = playlimit_sig
-        self.protocolver = "1.000"
+        self.playhistory = playhistory
         self.nearfull = nearfull
         self.nearfullsig = nearfull_sig
-        self.fixlogincnt = "0"
-        self.fixinterval = "5"
-        self.playhistory = playhistory
+        self.linelimit = 100
+        self.protocolver = float5.to_str(protocol_ver)
         # playhistory -> YYYYMM/C:...
         # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period
 
@@ -493,3 +786,66 @@ class AllnetRequestException(Exception):
     def __init__(self, message="") -> None:
         self.message = message
         super().__init__(self.message)
+
+class DLReport:
+    def __init__(self, data: Dict, report_type: DLIMG_TYPE) -> None:
+        self.serial = data.get("serial")
+        self.dfl = data.get("dfl")
+        self.wfl = data.get("wfl")
+        self.tsc = data.get("tsc")
+        self.tdsc = data.get("tdsc")
+        self.at = data.get("at")
+        self.ot = data.get("ot")
+        self.rt = data.get("rt")
+        self.as_ = data.get("as")
+        self.rf_state = DLI_STATUS.from_int(data.get("rf_state"))
+        self.gd = data.get("gd")
+        self.dav = data.get("dav")
+        self.wdav = data.get("wdav") # app only
+        self.dov = data.get("dov")
+        self.wdov = data.get("wdov") # app only
+        self.rep_type = report_type
+        self.err = ""
+    
+    def validate(self) -> bool:
+        if  self.serial is None:
+            self.err = "serial not provided"
+            return False
+        
+        if self.tsc is None:
+            self.err = "tsc not provided"
+            return False
+        
+        if self.tdsc is None:
+            self.err = "tdsc not provided"
+            return False
+        
+        if self.as_ is None:
+            self.err = "as not provided"
+            return False
+        
+        if self.rf_state is None:
+            self.err = "rf_state not provided"
+            return False
+        
+        if self.gd is None:
+            self.err = "gd not provided"
+            return False
+        
+        if self.dav is None:
+            self.err = "dav not provided"
+            return False
+        
+        if self.dov is None:
+            self.err = "dov not provided"
+            return False
+        
+        if (self.wdav is None or self.wdov is None) and self.rep_type == DLIMG_TYPE.app:
+            self.err = "wdav or wdov not provided in app image"
+            return False
+        
+        if (self.wdav is not None or self.wdov is not None) and self.rep_type == DLIMG_TYPE.opt:
+            self.err = "wdav or wdov provided in opt image"
+            return False
+        
+        return True

core/config.py

@@ -36,12 +36,36 @@ class ServerConfig:
             self.__config, "core", "server", "is_develop", default=True
         )
 
+    @property
+    def is_using_proxy(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "is_using_proxy", default=False
+        )
+
+    @property
+    def threading(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "threading", default=False
+        )
+
     @property
     def log_dir(self) -> str:
         return CoreConfig.get_config_field(
             self.__config, "core", "server", "log_dir", default="logs"
         )
 
+    @property
+    def check_arcade_ip(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "check_arcade_ip", default=False
+        )
+
+    @property
+    def strict_ip_checking(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "strict_ip_checking", default=False
+        )
+
 
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -66,6 +90,36 @@ class TitleConfig:
         return CoreConfig.get_config_field(
             self.__config, "core", "title", "port", default=8080
         )
+        
+    @property
+    def port_ssl(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "port_ssl", default=0
+        )
+
+    @property
+    def ssl_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_key", default="cert/title.key"
+        )
+
+    @property
+    def ssl_cert(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
+        )
+
+    @property
+    def reboot_start_time(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "reboot_start_time", default=""
+        )
+
+    @property
+    def reboot_end_time(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "reboot_end_time", default=""
+        )
 
 
 class DatabaseConfig:
@@ -105,7 +159,7 @@ class DatabaseConfig:
     @property
     def protocol(self) -> str:
         return CoreConfig.get_config_field(
-            self.__config, "core", "database", "type", default="mysql"
+            self.__config, "core", "database", "protocol", default="mysql"
         )
 
     @property
@@ -132,6 +186,19 @@ class DatabaseConfig:
             default=10000,
         )
 
+    @property
+    def sqlite_db_filename(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "sqlite_db_filename", default="aime.db"
+        )
+
+
+    @property
+    def enable_memcached(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "enable_memcached", default=True
+        )
+
     @property
     def memcached_host(self) -> str:
         return CoreConfig.get_config_field(
@@ -182,6 +249,12 @@ class AllnetConfig:
             self.__config, "core", "allnet", "port", default=80
         )
 
+    @property
+    def ip_check(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "ip_check", default=False
+        )
+
     @property
     def allow_online_updates(self) -> int:
         return CoreConfig.get_config_field(
@@ -248,6 +321,18 @@ class AimedbConfig:
             self.__config, "core", "aimedb", "key", default=""
         )
 
+    @property
+    def id_secret(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "id_secret", default=""
+        )
+
+    @property
+    def id_lifetime_seconds(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "id_lifetime_seconds", default=86400
+        )
+
 
 class MuchaConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:

core/data/cache.py

@@ -17,7 +17,7 @@ except ModuleNotFoundError:
 
 def cached(lifetime: int = 10, extra_key: Any = None) -> Callable:
     def _cached(func: Callable) -> Callable:
-        if has_mc:
+        if has_mc and (cfg and cfg.database.enable_memcached):
             hostname = "127.0.0.1"
             if cfg:
                 hostname = cfg.database.memcached_host

core/data/database.py

@@ -1,5 +1,8 @@
 import logging, coloredlogs
 from typing import Optional, Dict, List
+
+import alembic
+from alembic.config import Config
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
@@ -9,37 +12,61 @@ import secrets, string
 import bcrypt
 from hashlib import sha256
 
-from core.config import CoreConfig
+from core.config import CoreConfig, DatabaseConfig
 from core.data.schema import *
 from core.utils import Utils
 
 
+def build_db_url_from_config(database: DatabaseConfig):
+    if database.protocol == "sqlite":
+        return f"{database.protocol}:///{database.sqlite_db_filename}"
+
+    if database.sha2_password:
+        passwd = sha256(database.password.encode()).digest()
+        return f"{database.protocol}://{database.username}:{passwd.hex()}@{database.host}:{database.port}/{database.name}?charset=utf8mb4"
+    else:
+        return f"{database.protocol}://{database.username}:{database.password}@{database.host}:{database.port}/{database.name}?charset=utf8mb4"
+
+
 class Data:
+    current_schema_version = 6
+    engine = None
+    session = None
+    user = None
+    arcade = None
+    card = None
+    base = None
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
+        
+        self.__url = build_db_url_from_config(self.config.database)
 
-        if self.config.database.sha2_password:
-            passwd = sha256(self.config.database.password.encode()).digest()
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
-        else:
-            self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
+        if Data.engine is None:
+            Data.engine = create_engine(self.__url, pool_recycle=3600)
+            self.__engine = Data.engine
 
-        self.__engine = create_engine(self.__url, pool_recycle=3600)
-        session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
-        self.session = scoped_session(session)
+        if Data.session is None:
+            s = sessionmaker(bind=Data.engine, autoflush=True, autocommit=True)
+            Data.session = scoped_session(s)
 
-        self.user = UserData(self.config, self.session)
-        self.arcade = ArcadeData(self.config, self.session)
-        self.card = CardData(self.config, self.session)
-        self.base = BaseData(self.config, self.session)
-        self.current_schema_version = 4
+        if Data.user is None:
+            Data.user = UserData(self.config, self.session)
+        
+        if Data.arcade is None:
+            Data.arcade = ArcadeData(self.config, self.session)
+        
+        if Data.card is None:
+            Data.card = CardData(self.config, self.session)
+        
+        if Data.base is None:
+            Data.base = BaseData(self.config, self.session)
 
-        log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("database")
 
         # Prevent the logger from adding handlers multiple times
-        if not getattr(self.logger, "handler_set", None):
+        if not getattr(self.logger, "handler_set", None):            
+            log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
             fileHandler = TimedRotatingFileHandler(
                 "{0}/{1}.log".format(self.config.server.log_dir, "db"),
                 encoding="utf-8",
@@ -77,7 +104,7 @@ class Data:
                     game_mod.database(self.config)
                 metadata.create_all(self.__engine.connect())
 
-                self.base.set_schema_ver(
+                self.base.touch_schema_ver(
                     game_mod.current_schema_version, game_mod.game_codes[0]
                 )
 
@@ -146,7 +173,7 @@ class Data:
                         version = mod.current_schema_version
 
                     else:
-                        self.logger.warn(
+                        self.logger.warning(
                             f"current_schema_version not found for {folder}"
                         )
 
@@ -154,7 +181,7 @@ class Data:
                 version = self.current_schema_version
 
         if version is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"Could not determine latest version for {game}, please specify --version"
             )
 
@@ -237,7 +264,7 @@ class Data:
             self.logger.error(f"Failed to create card for owner with id {user_id}")
             return
 
-        self.logger.warn(
+        self.logger.warning(
             f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
         )
 
@@ -252,7 +279,7 @@ class Data:
             return
 
         if not should_force:
-            self.logger.warn(
+            self.logger.warning(
                 f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
                 f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
             )
@@ -290,7 +317,7 @@ class Data:
     def autoupgrade(self) -> None:
         all_game_versions = self.base.get_all_schema_vers()
         if all_game_versions is None:
-            self.logger.warn("Failed to get schema versions")
+            self.logger.warning("Failed to get schema versions")
             return
 
         all_games = Utils.get_all_titles()
@@ -333,3 +360,15 @@ class Data:
 
             if not failed:
                 self.base.set_schema_ver(latest_ver, game)
+    
+    def show_versions(self) -> None:
+        all_game_versions = self.base.get_all_schema_vers()
+        for ver in all_game_versions:
+            self.logger.info(f"{ver['game']} -> v{ver['version']}")
+
+    def alembic_upgrade(self, alembic_ini_path: str):
+        from alembic import command
+        alembic_cfg = alembic.config.Config(alembic_ini_path)
+        alembic_cfg.set_main_option('artemis_db_url', build_db_url_from_config(self.config.database))
+        command.upgrade(alembic_cfg, 'head')
+

core/data/schema/arcade.py

@@ -1,9 +1,9 @@
-from typing import Optional, Dict
-from sqlalchemy import Table, Column
+from typing import Optional, Dict, List
+from sqlalchemy import Table, Column, and_, or_
 from sqlalchemy.sql.schema import ForeignKey, PrimaryKeyConstraint
-from sqlalchemy.types import Integer, String, Boolean
-from sqlalchemy.sql import func, select
-from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.types import Integer, String, Boolean, JSON
+from sqlalchemy.sql import func, select, insert
+from sqlalchemy.engine import Row
 import re
 
 from core.data.schema.base import BaseData, metadata
@@ -21,6 +21,7 @@ arcade = Table(
     Column("city", String(255)),
     Column("region_id", Integer),
     Column("timezone", String(255)),
+    Column("ip", String(39)),
     mysql_charset="utf8mb4",
 )
 
@@ -39,7 +40,9 @@ machine = Table(
     Column("country", String(3)),  # overwrites if not null
     Column("timezone", String(255)),
     Column("ota_enable", Boolean),
+    Column("memo", String(255)),
     Column("is_cab", Boolean),
+    Column("data", JSON),
     mysql_charset="utf8mb4",
 )
 
@@ -65,7 +68,7 @@ arcade_owner = Table(
 
 
 class ArcadeData(BaseData):
-    def get_machine(self, serial: str = None, id: int = None) -> Optional[Dict]:
+    def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
         if serial is not None:
             serial = serial.replace("-", "")
             if len(serial) == 11:
@@ -130,12 +133,19 @@ class ArcadeData(BaseData):
                 f"Failed to update board id for machine {machine_id} -> {boardid}"
             )
 
-    def get_arcade(self, id: int) -> Optional[Dict]:
+    def get_arcade(self, id: int) -> Optional[Row]:
         sql = arcade.select(arcade.c.id == id)
         result = self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
+    
+    def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
+        sql = machine.select(machine.c.arcade == id)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
 
     def put_arcade(
         self,
@@ -165,7 +175,21 @@ class ArcadeData(BaseData):
             return None
         return result.lastrowid
 
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Dict]:
+    def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
+        sql = select(arcade).join(arcade_owner, arcade_owner.c.arcade == arcade.c.id).where(arcade_owner.c.user == user_id)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()
+    
+    def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
+        sql = select(arcade_owner.c.permissions).where(and_(arcade_owner.c.user == user_id, arcade_owner.c.arcade == arcade_id))
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
         sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
 
         result = self.execute(sql)
@@ -187,33 +211,21 @@ class ArcadeData(BaseData):
         return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
 
     def validate_keychip_format(self, serial: str) -> bool:
-        serial = serial.replace("-", "")
-        if len(serial) != 11 or len(serial) != 15:
-            self.logger.error(
-                f"Serial validate failed: Incorrect length for {serial} (len {len(serial)})"
-            )
+        if re.fullmatch(r"^A[0-9]{2}[E|X][-]?[0-9]{2}[A-HJ-NP-Z][0-9]{4}([0-9]{4})?$", serial) is None:
             return False
-
-        platform_code = serial[:4]
-        platform_rev = serial[4:6]
-        const_a = serial[6]
-        num = serial[7:11]
-        append = serial[11:15]
-
-        if re.match("A[7|6]\d[E|X][0|1][0|1|2]A\d{4,8}", serial) is None:
-            self.logger.error(f"Serial validate failed: {serial} failed regex")
-            return False
-
-        if len(append) != 0 or len(append) != 4:
-            self.logger.error(
-                f"Serial validate failed: {serial} had malformed append {append}"
-            )
-            return False
-
-        if len(num) != 4:
-            self.logger.error(
-                f"Serial validate failed: {serial} had malformed number {num}"
-            )
-            return False
-
+        
         return True
+
+    def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
+        sql = arcade.select(or_(arcade.c.name.like(f"%{name}%"), arcade.c.nickname.like(f"%{name}%")))
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
+        sql = arcade.select().where(arcade.c.ip == ip)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()

core/data/schema/base.py

@@ -1,15 +1,17 @@
 import json
 import logging
+import sqlite3
 from random import randrange
 from typing import Any, Optional, Dict, List
-from sqlalchemy.engine import Row
+
+import sqlalchemy.dialects.mysql
+from sqlalchemy.engine import Row, CursorResult
 from sqlalchemy.engine.cursor import CursorResult
 from sqlalchemy.engine.base import Connection
-from sqlalchemy.sql import text, func, select
-from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql import text, func, select, insert
+from sqlalchemy.exc import SQLAlchemyError, IntegrityError
 from sqlalchemy import MetaData, Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP, JSON
-from sqlalchemy.dialects.mysql import insert
 
 from core.config import CoreConfig
 
@@ -58,7 +60,7 @@ class BaseData:
             self.logger.error(f"UnicodeEncodeError error {e}")
             return None
 
-        except:
+        except Exception:
             try:
                 res = self.conn.execute(sql, opts)
 
@@ -70,7 +72,7 @@ class BaseData:
                 self.logger.error(f"UnicodeEncodeError error {e}")
                 return None
 
-            except:
+            except Exception:
                 self.logger.error(f"Unknown error")
                 raise
 
@@ -103,11 +105,68 @@ class BaseData:
 
         return row["version"]
 
-    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=ver)
+    def upsert(self, table: Table, data: Dict) -> CursorResult | None:
+        # fall back to the old behavior for mysql, and use current behavior for postgres
 
-        result = self.execute(conflict)
+        sql = self.dialect_insert(table).values(data)
+        unique_columns = get_unique_columns(table)
+        if len(unique_columns) > 0:
+            if self.conn.bind.name == "mysql":
+                conflict = sql.on_duplicate_key_update(**data)
+            elif self.conn.bind.name == "postgresql":
+                conflict = sql.on_conflict_do_update(
+                    index_elements=unique_columns,
+                    set_=data,
+                )
+
+            if self.conn.bind.name == "sqlite":
+                from sqlalchemy import exc
+                try:
+                    result = self.conn.execute(sql)
+                except (exc.IntegrityError, sqlite3.IntegrityError) as e:
+                    # run update query using the values from unique_columns
+                    from operator import and_
+                    conditions = [getattr(table.c, col) == data[col] for col in unique_columns]
+
+                    if len(conditions) > 1:
+                        from functools import reduce
+                        where_clause = reduce(and_, conditions)
+                    elif conditions:
+                        where_clause = conditions[0]
+
+                    upd = table.update().where(where_clause).values(data)
+                    self.logger.error(f"SQL post-conflict update ({', '.join(unique_columns)}): {str(upd)}")
+                    result = self.execute(upd)
+            else:
+                result = self.execute(conflict)
+        else:
+            result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to upsert data into {table.name}")
+            return None
+        return result
+
+    def dialect_insert(self, table: Table):
+        if self.conn.bind.name == "mysql":
+            return sqlalchemy.dialects.mysql.insert(table)
+        elif self.conn.bind.name == "postgresql":
+            return sqlalchemy.dialects.postgresql.insert(table)
+        elif self.conn.bind.name == "sqlite":
+            return sqlalchemy.dialects.sqlite.insert(table)
+        else:
+            raise Exception("Unknown dialect")
+
+    def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
+        result = self.upsert(schema_ver, dict(game=game, version=ver))
+        if result is None:
+            self.logger.error(
+                f"Failed to update schema version for game {game} (v{ver})"
+            )
+            return None
+        return result.lastrowid
+
+    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
+        result = self.upsert(schema_ver, dict(game=game, version=ver))
         if result is None:
             self.logger.error(
                 f"Failed to update schema version for game {game} (v{ver})"
@@ -145,9 +204,33 @@ class BaseData:
 
     def fix_bools(self, data: Dict) -> Dict:
         for k, v in data.items():
+            if k == "userName" or k == "teamName":
+                continue
             if type(v) == str and v.lower() == "true":
                 data[k] = True
             elif type(v) == str and v.lower() == "false":
                 data[k] = False
 
         return data
+
+
+unique_columns_cache = {}
+
+def get_unique_columns(table: Table):
+    global unique_columns_cache
+
+    # Check if the unique columns for this table are already cached
+    if table.name in unique_columns_cache:
+        return unique_columns_cache[table.name]
+
+    # Calculate the unique columns for this table
+    unique_columns = []
+    from sqlalchemy import UniqueConstraint, PrimaryKeyConstraint
+    has_unique_constraint = any(isinstance(constraint, UniqueConstraint) for constraint in table.constraints)
+    for constraint in table.constraints:
+        # if we do not have a unique constraint, use the primary key
+        if (not has_unique_constraint and isinstance(constraint, PrimaryKeyConstraint)) or isinstance(constraint, UniqueConstraint):
+            unique_columns.extend([column.name for column in constraint.columns])
+    unique_columns_cache[table.name] = unique_columns
+    print(unique_columns)
+    return unique_columns

core/data/schema/card.py

@@ -64,6 +64,27 @@ class CardData(BaseData):
 
         return int(card["user"])
 
+    def get_card_banned(self, access_code: str) -> Optional[bool]:
+        """
+        Given a 20 digit access code as a string, check if the card is banned
+        """
+        card = self.get_card_by_access_code(access_code)
+        if card is None:
+            return None
+        if card["is_banned"]:
+            return True
+        return False
+    def get_card_locked(self, access_code: str) -> Optional[bool]:
+        """
+        Given a 20 digit access code as a string, check if the card is locked
+        """
+        card = self.get_card_by_access_code(access_code)
+        if card is None:
+            return None
+        if card["is_locked"]:
+            return True
+        return False
+
     def delete_card(self, card_id: int) -> None:
         sql = aime_card.delete(aime_card.c.id == card_id)
 

core/data/schema/user.py

@@ -2,9 +2,7 @@ from enum import Enum
 from typing import Optional, List
 from sqlalchemy import Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP
-from sqlalchemy.sql import func
-from sqlalchemy.dialects.mysql import insert
-from sqlalchemy.sql import func, select
+from sqlalchemy.sql import func, select, insert
 from sqlalchemy.engine import Row
 import bcrypt
 
@@ -41,26 +39,21 @@ class UserData(BaseData):
         permission: int = 1,
     ) -> Optional[int]:
         if id is None:
-            sql = insert(aime_user).values(
+            result = self.upsert(aime_user, dict(
                 username=username,
                 email=email,
                 password=password,
                 permissions=permission,
-            )
+            ))
         else:
-            sql = insert(aime_user).values(
+            result = self.upsert(aime_user, dict(
                 id=id,
                 username=username,
                 email=email,
                 password=password,
                 permissions=permission,
-            )
+            ))
 
-        conflict = sql.on_duplicate_key_update(
-            username=username, email=email, password=password, permissions=permission
-        )
-
-        result = self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
@@ -79,6 +72,9 @@ class UserData(BaseData):
 
         if usr["password"] is None:
             return False
+        
+        if passwd is None or not passwd:
+            return False
 
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
@@ -104,3 +100,17 @@ class UserData(BaseData):
         if result is None:
             return None
         return result.fetchall()
+
+    def find_user_by_email(self, email: str) -> Row:
+        sql = select(aime_user).where(aime_user.c.email == email)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def find_user_by_username(self, username: str) -> List[Row]:
+        sql = aime_user.select(aime_user.c.username.like(f"%{username}%"))
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchall()

core/data/schema/versions/CORE_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine DROP COLUMN memo;
+ALTER TABLE machine DROP COLUMN is_blacklisted;
+ALTER TABLE machine DROP COLUMN `data`;

core/data/schema/versions/CORE_5_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade DROP COLUMN 'ip';

core/data/schema/versions/CORE_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE machine ADD memo varchar(255) NULL;
+ALTER TABLE machine ADD is_blacklisted tinyint(1) NULL;
+ALTER TABLE machine ADD `data` longtext NULL;

core/data/schema/versions/CORE_6_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE arcade ADD ip varchar(39) NULL;

core/data/schema/versions/SBZV_4_rollback.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    DROP cnp_cid,
+    DROP cnp_val,
+    DROP cnp_rr,
+    DROP cnp_sp,
+    DROP btn_se_eqp,
+    DROP sld_se_eqp,
+    DROP chn_sld_se_eqp,
+    DROP sldr_tch_se_eqp;

core/data/schema/versions/SBZV_5_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE diva_profile
+    DROP skn_eqp;

core/data/schema/versions/SBZV_5_upgrade.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    ADD cnp_cid INT NOT NULL DEFAULT -1,
+    ADD cnp_val INT NOT NULL DEFAULT -1,
+    ADD cnp_rr INT NOT NULL DEFAULT -1,
+    ADD cnp_sp VARCHAR(255) NOT NULL DEFAULT "",
+    ADD btn_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD chn_sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sldr_tch_se_eqp INT NOT NULL DEFAULT -1;

core/data/schema/versions/SBZV_6_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE diva_profile
+    ADD skn_eqp INT NOT NULL DEFAULT 0;

core/data/schema/versions/SDBT_3_rollback.sql

@@ -0,0 +1,30 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    DROP COLUMN regionId,
+    DROP COLUMN machineType;
+
+ALTER TABLE chuni_static_events
+    DROP COLUMN startDate;
+
+ALTER TABLE chuni_profile_data
+    DROP COLUMN rankUpChallengeResults;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN presetId id INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD PRIMARY KEY(id);
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_uk UNIQUE(id, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY(presetId)
+    REFERENCES chuni_static_login_bonus_preset(id) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_rollback.sql

@@ -0,0 +1,12 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+  CHANGE COLUMN isClear isClear TINYINT(1) NULL DEFAULT NULL;
+
+ALTER TABLE chuni_score_best
+  CHANGE COLUMN isSuccess isSuccess TINYINT(1) NULL DEFAULT NULL ;
+
+ALTER TABLE chuni_score_playlog
+  DROP COLUMN ticketId;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_upgrade.sql

@@ -0,0 +1,29 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    ADD COLUMN regionId INT,
+    ADD COLUMN machineType INT;
+
+ALTER TABLE chuni_static_events
+    ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();
+
+ALTER TABLE chuni_profile_data
+    ADD COLUMN rankUpChallengeResults JSON;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN id presetId INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP INDEX chuni_static_login_bonus_preset_uk;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_pk PRIMARY KEY (presetId, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY (presetId, version)
+    REFERENCES chuni_static_login_bonus_preset(presetId, version) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_5_upgrade.sql

@@ -0,0 +1,12 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+  CHANGE COLUMN isClear isClear TINYINT(6) NULL DEFAULT NULL;
+
+ALTER TABLE chuni_score_best
+  CHANGE COLUMN isSuccess isSuccess INT(11) NULL DEFAULT NULL ;
+
+ALTER TABLE chuni_score_playlog
+  ADD COLUMN ticketId INT(11) NULL AFTER machineType;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDDT_4_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+DROP COLUMN startDate;

core/data/schema/versions/SDDT_5_rollback.sql

@@ -0,0 +1,22 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+ALTER TABLE ongeki_user_event_point DROP COLUMN version;
+ALTER TABLE ongeki_user_event_point DROP COLUMN rank;
+ALTER TABLE ongeki_user_event_point DROP COLUMN type;
+ALTER TABLE ongeki_user_event_point DROP COLUMN date;
+
+ALTER TABLE ongeki_user_tech_event DROP COLUMN version;
+
+ALTER TABLE ongeki_user_mission_point DROP COLUMN version;
+
+ALTER TABLE ongeki_static_events DROP COLUMN endDate;
+
+DROP TABLE ongeki_tech_event_ranking;
+DROP TABLE ongeki_static_music_ranking_list;
+DROP TABLE ongeki_static_rewards;
+DROP TABLE ongeki_static_present_list;
+DROP TABLE ongeki_static_tech_music;
+DROP TABLE ongeki_static_client_testmode;
+DROP TABLE ongeki_static_game_point;
+
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDDT_5_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();

core/data/schema/versions/SDDT_6_upgrade.sql

@@ -0,0 +1,98 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+ALTER TABLE ongeki_user_event_point ADD COLUMN version INTEGER NOT NULL;
+ALTER TABLE ongeki_user_event_point ADD COLUMN rank INTEGER;
+ALTER TABLE ongeki_user_event_point ADD COLUMN type INTEGER NOT NULL;
+ALTER TABLE ongeki_user_event_point ADD COLUMN date VARCHAR(25);
+
+ALTER TABLE ongeki_user_tech_event ADD COLUMN version INTEGER NOT NULL;
+
+ALTER TABLE ongeki_user_mission_point ADD COLUMN version INTEGER NOT NULL;
+
+ALTER TABLE ongeki_static_events ADD COLUMN endDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
+
+CREATE TABLE ongeki_tech_event_ranking (
+	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+	user INT NOT NULL,
+	version INT NOT NULL,
+	date VARCHAR(25),
+	eventId INT NOT NULL,
+	rank INT,
+	totalPlatinumScore INT NOT NULL,
+	totalTechScore INT NOT NULL,
+	UNIQUE KEY ongeki_tech_event_ranking_uk (user, eventId),
+	CONSTRAINT ongeki_tech_event_ranking_ibfk1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE TABLE ongeki_static_music_ranking_list (
+	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+	version INT NOT NULL,
+	musicId INT NOT NULL,
+	point INT NOT NULL,
+	userName VARCHAR(255),
+	UNIQUE KEY ongeki_static_music_ranking_list_uk (version, musicId)
+);
+
+CREATE TABLE ongeki_static_rewards (
+	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+	version INT NOT NULL,
+	rewardId INT NOT NULL,
+	rewardName VARCHAR(255) NOT NULL,
+	itemKind INT NOT NULL,
+	itemId INT NOT NULL,
+	UNIQUE KEY ongeki_tech_event_ranking_uk (version, rewardId)
+);
+
+CREATE TABLE ongeki_static_present_list (
+	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+	version INT NOT NULL,
+	presentId INT NOT NULL,
+	presentName VARCHAR(255) NOT NULL,
+	rewardId INT NOT NULL,
+	stock INT NOT NULL,
+	message VARCHAR(255),
+	startDate VARCHAR(25) NOT NULL,
+	endDate VARCHAR(25) NOT NULL,
+	UNIQUE KEY ongeki_static_present_list_uk (version, presentId, rewardId)
+);
+
+CREATE TABLE ongeki_static_tech_music (
+	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+	version INT NOT NULL,
+	eventId INT NOT NULL,
+	musicId INT NOT NULL,
+	level INT NOT NULL,
+	UNIQUE KEY ongeki_static_tech_music_uk (version, musicId, eventId)
+);
+
+CREATE TABLE ongeki_static_client_testmode (
+	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+	regionId INT NOT NULL,
+	placeId INT NOT NULL,
+	clientId VARCHAR(11) NOT NULL,
+	updateDate TIMESTAMP NOT NULL,
+	isDelivery BOOLEAN NOT NULL,
+	groupId INT NOT NULL,
+	groupRole INT NOT NULL,
+	continueMode INT NOT NULL,
+	selectMusicTime INT NOT NULL,
+	advertiseVolume INT NOT NULL,
+	eventMode INT NOT NULL,
+	eventMusicNum INT NOT NULL,
+	patternGp INT NOT NULL,
+	limitGp INT NOT NULL,
+	maxLeverMovable INT NOT NULL,
+	minLeverMovable INT NOT NULL,
+	UNIQUE KEY ongeki_static_client_testmode_uk (clientId)
+);
+
+CREATE TABLE ongeki_static_game_point (
+	id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+	type INT NOT NULL,
+	cost INT NOT NULL,
+	startDate VARCHAR(25) NOT NULL DEFAULT "2000-01-01 05:00:00.0",
+	endDate VARCHAR(25) NOT NULL DEFAULT "2099-01-01 05:00:00.0",
+	UNIQUE KEY ongeki_static_game_point_uk (type)
+);
+
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDEZ_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP DEFAULT "2018-01-01 00:00:00.0",
+    CHANGE COLUMN endDate endDate TIMESTAMP DEFAULT "2038-01-01 00:00:00.0";

core/data/schema/versions/SDEZ_5_rollback.sql

@@ -0,0 +1,78 @@
+DELETE FROM mai2_static_event WHERE version < 13;
+UPDATE mai2_static_event SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_music WHERE version < 13;
+UPDATE mai2_static_music SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_ticket WHERE version < 13;
+UPDATE mai2_static_ticket SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_cards WHERE version < 13;
+UPDATE mai2_static_cards SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_detail WHERE version < 13;
+UPDATE mai2_profile_detail SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_extend WHERE version < 13;
+UPDATE mai2_profile_extend SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_option WHERE version < 13;
+UPDATE mai2_profile_option SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_ghost WHERE version < 13;
+UPDATE mai2_profile_ghost SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_rating WHERE version < 13;
+UPDATE mai2_profile_rating SET version = version - 13 WHERE version >= 13;
+
+DROP TABLE maimai_score_best;
+DROP TABLE maimai_playlog;
+DROP TABLE maimai_profile_detail;
+DROP TABLE maimai_profile_option;
+DROP TABLE maimai_profile_web_option;
+DROP TABLE maimai_profile_grade_status;
+
+ALTER TABLE mai2_item_character DROP COLUMN point;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NOT NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NOT NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NOT NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NOT NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NOT NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN `rank` int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NOT NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;

core/data/schema/versions/SDEZ_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CHANGE COLUMN endDate endDate TIMESTAMP NOT NULL;

core/data/schema/versions/SDEZ_6_rollback.sql

@@ -0,0 +1 @@
+DROP TABLE aime.mai2_profile_consec_logins;

core/data/schema/versions/SDEZ_6_upgrade.sql

@@ -0,0 +1,62 @@
+UPDATE mai2_static_event SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_music SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_ticket SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_cards SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_detail  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_extend  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_option  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_ghost  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_rating  SET version = version + 13 WHERE version < 1000;
+
+ALTER TABLE mai2_item_character ADD point int(11) NULL;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN `rank` int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;

core/data/schema/versions/SDEZ_7_rollback.sql

@@ -0,0 +1,10 @@
+ALTER TABLE mai2_profile_detail
+    DROP COLUMN mapStock;
+
+ALTER TABLE mai2_profile_extend
+    DROP COLUMN selectResultScoreViewType;
+
+ALTER TABLE mai2_profile_option
+    DROP COLUMN outFrameType,
+    DROP COLUMN touchVolume,
+    DROP COLUMN breakSlideVolume;

core/data/schema/versions/SDEZ_7_upgrade.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `mai2_profile_consec_logins` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `logins` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `mai2_profile_consec_logins_uk` (`user`,`version`),
+  CONSTRAINT `mai2_profile_consec_logins_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

core/data/schema/versions/SDEZ_8_upgrade.sql

@@ -0,0 +1,10 @@
+ALTER TABLE mai2_profile_detail
+    ADD mapStock INT NULL AFTER playCount;
+
+ALTER TABLE mai2_profile_extend
+    ADD selectResultScoreViewType INT NULL AFTER selectResultDetails;
+
+ALTER TABLE mai2_profile_option
+    ADD outFrameType INT NULL AFTER dispCenter,
+    ADD touchVolume INT NULL AFTER slideVolume,
+    ADD breakSlideVolume INT NULL AFTER slideVolume;

core/frontend.py

@@ -9,6 +9,9 @@ from zope.interface import Interface, Attribute, implementer
 from twisted.python.components import registerAdapter
 import jinja2
 import bcrypt
+import re
+from enum import Enum
+from urllib import parse
 
 from core import CoreConfig, Utils
 from core.data import Data
@@ -18,7 +21,15 @@ class IUserSession(Interface):
     userId = Attribute("User's ID")
     current_ip = Attribute("User's current ip address")
     permissions = Attribute("User's permission level")
+    ongeki_version = Attribute("User's selected Ongeki Version")
 
+class PermissionOffset(Enum):
+    USER = 0 # Regular user
+    USERMOD = 1 # Can moderate other users
+    ACMOD = 2 # Can add arcades and cabs
+    SYSADMIN = 3 # Can change settings
+    # 4 - 6 reserved for future use
+    OWNER = 7 # Can do anything
 
 @implementer(IUserSession)
 class UserSession(object):
@@ -26,6 +37,7 @@ class UserSession(object):
         self.userId = 0
         self.current_ip = "0.0.0.0"
         self.permissions = 0
+        self.ongeki_version = 7
 
 
 class FrontendServlet(resource.Resource):
@@ -80,6 +92,9 @@ class FrontendServlet(resource.Resource):
         self.environment.globals["game_list"] = self.game_list
         self.putChild(b"gate", FE_Gate(cfg, self.environment))
         self.putChild(b"user", FE_User(cfg, self.environment))
+        self.putChild(b"sys", FE_System(cfg, self.environment))
+        self.putChild(b"arcade", FE_Arcade(cfg, self.environment))
+        self.putChild(b"cab", FE_Machine(cfg, self.environment))
         self.putChild(b"game", fe_game)
 
         self.logger.info(
@@ -130,7 +145,7 @@ class FE_Gate(FE_Base):
         if b"e" in request.args:
             try:
                 err = int(request.args[b"e"][0].decode())
-            except:
+            except Exception:
                 err = 0
 
         else:
@@ -154,6 +169,7 @@ class FE_Gate(FE_Base):
                 passwd = None
 
             uid = self.data.card.get_user_id_from_card(access_code)
+            user = self.data.user.get_user(uid)
             if uid is None:
                 return redirectTo(b"/gate?e=1", request)
 
@@ -175,6 +191,7 @@ class FE_Gate(FE_Base):
             usr_sesh = IUserSession(sesh)
             usr_sesh.userId = uid
             usr_sesh.current_ip = ip
+            usr_sesh.permissions = user['permissions']
 
             return redirectTo(b"/user", request)
 
@@ -182,7 +199,7 @@ class FE_Gate(FE_Base):
             access_code: str = request.args[b"access_code"][0].decode()
             username: str = request.args[b"username"][0]
             email: str = request.args[b"email"][0].decode()
-            passwd: str = request.args[b"passwd"][0]
+            passwd: bytes = request.args[b"passwd"][0]
 
             uid = self.data.card.get_user_id_from_card(access_code)
             if uid is None:
@@ -192,12 +209,12 @@ class FE_Gate(FE_Base):
             hashed = bcrypt.hashpw(passwd, salt)
 
             result = self.data.user.create_user(
-                uid, username, email, hashed.decode(), 1
+                uid, username, email.lower(), hashed.decode(), 1
             )
             if result is None:
                 return redirectTo(b"/gate?e=3", request)
 
-            if not self.data.user.check_password(uid, passwd.encode()):
+            if not self.data.user.check_password(uid, passwd):
                 return redirectTo(b"/gate", request)
 
             return redirectTo(b"/user", request)
@@ -210,26 +227,172 @@ class FE_Gate(FE_Base):
             return redirectTo(b"/gate?e=2", request)
 
         ac = request.args[b"ac"][0].decode()
+        card = self.data.card.get_card_by_access_code(ac)
+        if card is None:
+            return redirectTo(b"/gate?e=1", request)
+        
+        user = self.data.user.get_user(card['user'])
+        if user is None:
+            self.logger.warning(f"Card {ac} exists with no/invalid associated user ID {card['user']}")
+            return redirectTo(b"/gate?e=0", request)
+
+        if user['password'] is not None:
+            return redirectTo(b"/gate?e=1", request)
 
         template = self.environment.get_template("core/frontend/gate/create.jinja")
         return template.render(
             title=f"{self.core_config.server.name} | Create User",
             code=ac,
-            sesh={"userId": 0},
+            sesh={"userId": 0, "permissions": 0},
         ).encode("utf-16")
 
 
 class FE_User(FE_Base):
     def render_GET(self, request: Request):
+        uri = request.uri.decode()
         template = self.environment.get_template("core/frontend/user/index.jinja")
 
         sesh: Session = request.getSession()
         usr_sesh = IUserSession(sesh)
         if usr_sesh.userId == 0:
             return redirectTo(b"/gate", request)
+        
+        m = re.match("\/user\/(\d*)", uri)
+        
+        if m is not None:            
+            usrid = m.group(1)
+            if usr_sesh.permissions < 1 << PermissionOffset.USERMOD.value or not usrid == usr_sesh.userId:
+                return redirectTo(b"/user", request)
+        
+        else:
+            usrid = usr_sesh.userId
+        
+        user = self.data.user.get_user(usrid)
+        if user is None:
+            return redirectTo(b"/user", request)
+        
+        cards = self.data.card.get_user_cards(usrid)
+        arcades = self.data.arcade.get_arcades_managed_by_user(usrid)
+        
+        card_data = []
+        arcade_data = []
+
+        for c in cards:
+            if c['is_locked']:
+                status = 'Locked'
+            elif c['is_banned']:
+                status = 'Banned'
+            else:
+                status = 'Active'
+            
+            card_data.append({'access_code': c['access_code'], 'status': status})
+        
+        for a in arcades:
+            arcade_data.append({'id': a['id'], 'name': a['name']})
 
         return template.render(
-            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh)
+            title=f"{self.core_config.server.name} | Account", 
+            sesh=vars(usr_sesh), 
+            cards=card_data, 
+            username=user['username'],
+            arcades=arcade_data
+        ).encode("utf-16")
+
+    def render_POST(self, request: Request):
+        pass
+
+
+class FE_System(FE_Base):
+    def render_GET(self, request: Request):
+        uri = request.uri.decode()
+        template = self.environment.get_template("core/frontend/sys/index.jinja")
+        usrlist: List[Dict] = []
+        aclist: List[Dict] = []
+        cablist: List[Dict] = []
+
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if usr_sesh.userId == 0 or usr_sesh.permissions < 1 << PermissionOffset.USERMOD.value:
+            return redirectTo(b"/gate", request)
+        
+        if uri.startswith("/sys/lookup.user?"):
+            uri_parse = parse.parse_qs(uri.replace("/sys/lookup.user?", "")) # lop off the first bit
+            uid_search = uri_parse.get("usrId")
+            email_search = uri_parse.get("usrEmail")
+            uname_search = uri_parse.get("usrName")
+
+            if uid_search is not None:
+                u = self.data.user.get_user(uid_search[0])
+                if u is not None:
+                    usrlist.append(u._asdict())
+
+            elif email_search is not None:
+                u = self.data.user.find_user_by_email(email_search[0])
+                if u is not None:
+                    usrlist.append(u._asdict())
+
+            elif uname_search is not None:
+                ul = self.data.user.find_user_by_username(uname_search[0])
+                for u in ul:
+                    usrlist.append(u._asdict())
+
+        elif uri.startswith("/sys/lookup.arcade?"):
+            uri_parse = parse.parse_qs(uri.replace("/sys/lookup.arcade?", "")) # lop off the first bit
+            ac_id_search = uri_parse.get("arcadeId")
+            ac_name_search = uri_parse.get("arcadeName")
+            ac_user_search = uri_parse.get("arcadeUser")
+            ac_ip_search = uri_parse.get("arcadeIp")
+
+            if ac_id_search is not None:
+                u = self.data.arcade.get_arcade(ac_id_search[0])
+                if u is not None:
+                    aclist.append(u._asdict())
+
+            elif ac_name_search is not None:
+                ul = self.data.arcade.get_arcade_by_name(ac_name_search[0])
+                if ul is not None:
+                    for u in ul:
+                        aclist.append(u._asdict())
+
+            elif ac_user_search is not None:
+                ul = self.data.arcade.get_arcades_managed_by_user(ac_user_search[0])
+                if ul is not None:
+                    for u in ul:
+                        aclist.append(u._asdict())
+
+            elif ac_ip_search is not None:
+                ul = self.data.arcade.get_arcades_by_ip(ac_ip_search[0])
+                if ul is not None:
+                    for u in ul:
+                        aclist.append(u._asdict())
+        
+        elif uri.startswith("/sys/lookup.cab?"):
+            uri_parse = parse.parse_qs(uri.replace("/sys/lookup.cab?", "")) # lop off the first bit
+            cab_id_search = uri_parse.get("cabId")
+            cab_serial_search = uri_parse.get("cabSerial")
+            cab_acid_search = uri_parse.get("cabAcId")
+
+            if cab_id_search is not None:
+                u = self.data.arcade.get_machine(id=cab_id_search[0])
+                if u is not None:
+                    cablist.append(u._asdict())
+
+            elif cab_serial_search is not None:
+                u = self.data.arcade.get_machine(serial=cab_serial_search[0])
+                if u is not None:
+                    cablist.append(u._asdict())
+
+            elif cab_acid_search is not None:
+                ul = self.data.arcade.get_arcade_machines(cab_acid_search[0])
+                for u in ul:
+                    cablist.append(u._asdict())
+
+        return template.render(
+            title=f"{self.core_config.server.name} | System", 
+            sesh=vars(usr_sesh), 
+            usrlist=usrlist,
+            aclist=aclist,
+            cablist=cablist,
         ).encode("utf-16")
 
 
@@ -244,3 +407,54 @@ class FE_Game(FE_Base):
 
     def render_GET(self, request: Request) -> bytes:
         return redirectTo(b"/user", request)
+
+
+class FE_Arcade(FE_Base):
+    def render_GET(self, request: Request):
+        uri = request.uri.decode()
+        template = self.environment.get_template("core/frontend/arcade/index.jinja")
+        managed = []
+        
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if usr_sesh.userId == 0:
+            return redirectTo(b"/gate", request)
+        
+        m = re.match("\/arcade\/(\d*)", uri)
+        
+        if m is not None:
+            arcadeid = m.group(1)
+            perms = self.data.arcade.get_manager_permissions(usr_sesh.userId, arcadeid)
+            arcade = self.data.arcade.get_arcade(arcadeid)
+
+            if perms is None:
+                perms = 0
+        
+        else:
+            return redirectTo(b"/user", request)
+        
+        return template.render(
+            title=f"{self.core_config.server.name} | Arcade", 
+            sesh=vars(usr_sesh),
+            error=0,
+            perms=perms,
+            arcade=arcade._asdict()
+        ).encode("utf-16")
+
+
+class FE_Machine(FE_Base):
+    def render_GET(self, request: Request):
+        uri = request.uri.decode()
+        template = self.environment.get_template("core/frontend/machine/index.jinja")
+        
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if usr_sesh.userId == 0:
+            return redirectTo(b"/gate", request)
+        
+        return template.render(
+            title=f"{self.core_config.server.name} | Machine", 
+            sesh=vars(usr_sesh), 
+            arcade={},
+            error=0,
+        ).encode("utf-16")

core/frontend/arcade/index.jinja

@@ -0,0 +1,4 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+<h1>{{ arcade.name }}</h1>
+{% endblock content %}

core/frontend/index.jinja

@@ -4,6 +4,7 @@
         <title>{{ title }}</title>
         <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
         <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4" crossorigin="anonymous"></script>
+        <script src="https://cdn.jsdelivr.net/npm/jquery@3.2.1/dist/jquery.min.js" integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4=" crossorigin="anonymous"></script>
         <style>
             html {
                 background-color: #181a1b !important;
@@ -77,6 +78,9 @@
                 margin-bottom: 10px;
                 width: 15%;
             }
+            .modal-content {
+                background-color: #181a1b;
+            }
         </style>
     </head>
     <body>

core/frontend/machine/index.jinja

@@ -0,0 +1,5 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+{% include "core/frontend/widgets/err_banner.jinja" %}
+<h1>Machine Management</h1>
+{% endblock content %}

core/frontend/sys/index.jinja

@@ -0,0 +1,103 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+<h1>System Management</h1>
+
+<div class="row" id="rowForm">
+    {% if sesh.permissions >= 2 %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
+            <h3>User Search</h3>
+            <div class="form-group">
+                <label for="usrId">User ID</label>
+                <input type="number" class="form-control" id="usrId" name="usrId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrName">Username</label>
+                <input type="text" class="form-control" id="usrName" name="usrName">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrEmail">Email address</label>
+                <input type="email" class="form-control" id="usrEmail" name="usrEmail" aria-describedby="emailHelp">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+    {% if sesh.permissions >= 4 %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="arcadeLookup" name="arcadeLookup" action="/sys/lookup.arcade" class="form-inline" >
+            <h3>Arcade Search</h3>
+            <div class="form-group">
+                <label for="arcadeId">Arcade ID</label>
+                <input type="number" class="form-control" id="arcadeId" name="arcadeId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="arcadeName">Arcade Name</label>
+                <input type="text" class="form-control" id="arcadeName" name="arcadeName">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="arcadeUser">Owner User ID</label>
+                <input type="number" class="form-control" id="arcadeUser" name="arcadeUser">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="arcadeIp">Assigned IP Address</label>
+                <input type="text" class="form-control" id="arcadeIp" name="arcadeIp">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="cabLookup" name="cabLookup" action="/sys/lookup.cab" class="form-inline" >
+            <h3>Machine Search</h3>
+            <div class="form-group">
+                <label for="cabId">Machine ID</label>
+                <input type="number" class="form-control" id="cabId" name="cabId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="cabSerial">Machine Serial</label>
+                <input type="text" class="form-control" id="cabSerial" name="cabSerial">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="cabAcId">Arcade ID</label>
+                <input type="number" class="form-control" id="cabAcId" name="cabAcId">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowResult" style="margin: 10px;">
+    {% if sesh.permissions >= 2 %}
+    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for usr in usrlist %}
+        <a href=/user/{{ usr.id }}><pre>{{ usr.id }} | {{ usr.username if usr.username != None else "<i>No Name Set</i>"}}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+    {% if sesh.permissions >= 4 %}
+    <div id="arcadeSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for ac in aclist %}
+        <pre><a href=/arcade/{{ ac.id }}>{{ ac.id }} | {{ ac.name if ac.name != None else "<i>No Name Set</i>" }} | {{ ac.ip if ac.ip != None else "<i>No IP Assigned</i>"}}</pre></a>
+        {% endfor %}
+    </div
+    ><div id="cabSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for cab in cablist %}
+        <a href=/cab/{{ cab.id }}><pre>{{ cab.id }} | {{ cab.game if cab.game != None else "<i>ANY </i>" }} | {{ cab.serial }}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowAdd">
+
+</div>
+{% endblock content %}

core/frontend/user/index.jinja

@@ -1,4 +1,41 @@
 {% extends "core/frontend/index.jinja" %}
 {% block content %}
-<h1>testing</h1>
+<h1>Management for {{ username }}</h1>
+<h2>Cards <button class="btn btn-success" data-bs-toggle="modal" data-bs-target="#card_add">Add</button></h2>
+<ul style="font-size: 20px;">
+{% for c in cards %}
+<li>{{ c.access_code }}: {{ c.status }}&nbsp;{% if c.status == 'Active'%}<button class="btn-warning btn">Lock</button>{% elif c.status == 'Locked' %}<button class="btn-warning btn">Unlock</button>{% endif %}&nbsp;<button class="btn-danger btn">Delete</button></li>
+{% endfor %}
+</ul>
+
+{% if arcades is defined %}
+<h2>Arcades</h2>
+  <ul style="font-size: 20px;">
+  {% for a in arcades %}
+    <li><a href=/arcade/{{ a.id }}>{{ a.name }}</a></li>
+  {% endfor %}
+  </ul>
+{% endif %}
+
+<div class="modal fade" id="card_add" tabindex="-1" aria-labelledby="card_add_label" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h1 class="modal-title fs-5" id="card_add_label">Add Card</h1>
+        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+      </div>
+      <div class="modal-body">
+        HOW TO:<br>
+        Scan your card on any networked game and press the "View Access Code" button (varies by game) and enter the 20 digit code below.<br>
+        !!FOR AMUSEIC CARDS: DO NOT ENTER THE CODE SHOWN ON THE BACK OF THE CARD ITSELF OR IT WILL NOT WORK!!
+        <p /><label for="card_add_frm_access_code">Access Code:&nbsp;</label><input id="card_add_frm_access_code" maxlength="20" type="text" required> 
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-primary">Add</button>
+        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 {% endblock content %}

core/frontend/widgets/err_banner.jinja

@@ -7,6 +7,10 @@ Card not registered, or wrong password
 Missing or malformed access code
 {% elif error == 3 %}
 Failed to create user
+{% elif error == 4 %}
+Arcade not found
+{% elif error == 5 %}
+Machine not found
 {% else %}
 An unknown error occoured
 {% endif %}

core/frontend/widgets/topbar.jinja

@@ -4,11 +4,14 @@
 <div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
     <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
     {% for game in game_list %}    
-    <a href=game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
+    <a href=/game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
     {% endfor %}
 </div>
 </div>
 <div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
+    {% if sesh is defined and sesh["permissions"] >= 2 %}
+    <a href="/sys"><button class="btn btn-primary">System</button></a>
+    {% endif %}
     {% if sesh is defined and sesh["userId"] > 0 %}
     <a href="/user"><button class="btn btn-primary">Account</button></a>
     {% else %}

core/mucha.py

@@ -4,17 +4,18 @@ from logging.handlers import TimedRotatingFileHandler
 from twisted.web import resource
 from twisted.web.http import Request
 from datetime import datetime
+from Crypto.Cipher import Blowfish
 import pytz
 
-from core import CoreConfig
-from core.utils import Utils
-
+from .config import CoreConfig
+from .utils import Utils
+from .title import TitleServlet
 
 class MuchaServlet:
+    mucha_registry: List[str] = []
     def __init__(self, cfg: CoreConfig, cfg_dir: str) -> None:
         self.config = cfg
         self.config_dir = cfg_dir
-        self.mucha_registry: List[str] = []
 
         self.logger = logging.getLogger("mucha")
         log_fmt_str = "[%(asctime)s] Mucha | %(levelname)s | %(message)s"
@@ -33,14 +34,12 @@ class MuchaServlet:
         self.logger.addHandler(fileHandler)
         self.logger.addHandler(consoleHandler)
 
-        self.logger.setLevel(logging.INFO)
-        coloredlogs.install(level=logging.INFO, logger=self.logger, fmt=log_fmt_str)
+        self.logger.setLevel(cfg.mucha.loglevel)
+        coloredlogs.install(level=cfg.mucha.loglevel, logger=self.logger, fmt=log_fmt_str)
 
-        all_titles = Utils.get_all_titles()
-
-        for _, mod in all_titles.items():
-            if hasattr(mod, "index") and hasattr(mod.index, "get_mucha_info"):
-                enabled, game_cd = mod.index.get_mucha_info(
+        for _, mod in TitleServlet.title_registry.items():
+            if hasattr(mod, "get_mucha_info"):
+                enabled, game_cd = mod.get_mucha_info(
                     self.config, self.config_dir
                 )
                 if enabled:
@@ -56,17 +55,24 @@ class MuchaServlet:
             self.logger.error(
                 f"Error processing mucha request {request.content.getvalue()}"
             )
-            return b""
+            return b"RESULTS=000"
 
         req = MuchaAuthRequest(req_dict)
+        self.logger.info(f"Boardauth request from {client_ip} for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(f"Boardauth request from {client_ip} for {req.gameVer}")
 
         if req.gameCd not in self.mucha_registry:
-            self.logger.warn(f"Unknown gameCd {req.gameCd}")
-            return b""
+            self.logger.warning(f"Unknown gameCd {req.gameCd}")
+            return b"RESULTS=000"
 
         # TODO: Decrypt S/N
+        b_key = b""
+        for x in range(8):
+            b_key += req.sendDate[(x - 1) & 7].encode()
+
+        cipher = Blowfish.new(b_key, Blowfish.MODE_ECB)
+        sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))
+        self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}")
 
         resp = MuchaAuthResponse(
             f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}"
@@ -84,22 +90,37 @@ class MuchaServlet:
             self.logger.error(
                 f"Error processing mucha request {request.content.getvalue()}"
             )
-            return b""
+            return b"RESULTS=000"
 
         req = MuchaUpdateRequest(req_dict)
+        self.logger.info(f"Updatecheck request from {client_ip} for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(f"Updatecheck request from {client_ip} for {req.gameVer}")
 
         if req.gameCd not in self.mucha_registry:
-            self.logger.warn(f"Unknown gameCd {req.gameCd}")
-            return b""
+            self.logger.warning(f"Unknown gameCd {req.gameCd}")
+            return b"RESULTS=000"
 
-        resp = MuchaUpdateResponseStub(req.gameVer)
+        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}")
 
         self.logger.debug(f"Mucha response {vars(resp)}")
 
         return self.mucha_postprocess(vars(resp))
 
+    def handle_dlstate(self, request: Request, _: Dict) -> bytes:
+        req_dict = self.mucha_preprocess(request.content.getvalue())
+        client_ip = Utils.get_ip_addr(request)
+
+        if req_dict is None:
+            self.logger.error(
+                f"Error processing mucha request {request.content.getvalue()}"
+            )
+            return b""
+        
+        req = MuchaDownloadStateRequest(req_dict)
+        self.logger.info(f"DownloadState request from {client_ip} for {req.gameCd} -> {req.updateVer}")        
+        self.logger.debug(f"request {vars(req)}")
+        return b"RESULTS=001"
+
     def mucha_preprocess(self, data: bytes) -> Optional[Dict]:
         try:
             ret: Dict[str, Any] = {}
@@ -111,19 +132,17 @@ class MuchaServlet:
 
             return ret
 
-        except:
+        except Exception:
             self.logger.error(f"Error processing mucha request {data}")
             return None
 
     def mucha_postprocess(self, data: dict) -> Optional[bytes]:
         try:
-            urlencode = ""
-            for k, v in data.items():
-                urlencode += f"{k}={v}&"
+            urlencode = "&".join(f"{k}={v}" for k, v in data.items())
 
             return urlencode.encode()
 
-        except:
+        except Exception:
             self.logger.error("Error processing mucha response")
             return None
 
@@ -202,22 +221,121 @@ class MuchaUpdateRequest:
 
 class MuchaUpdateResponse:
     def __init__(self, game_ver: str, mucha_url: str) -> None:
-        self.RESULTS = "001"
+        self.RESULTS = "001"        
+        self.EXE_VER = game_ver
+
         self.UPDATE_VER_1 = game_ver
-        self.UPDATE_URL_1 = f"https://{mucha_url}/updUrl1/"
-        self.UPDATE_SIZE_1 = "0"
-        self.UPDATE_CRC_1 = "0000000000000000"
-        self.CHECK_URL_1 = f"https://{mucha_url}/checkUrl/"
-        self.EXE_VER_1 = game_ver
+        self.UPDATE_URL_1 = f"http://{mucha_url}/updUrl1/"
+        self.UPDATE_SIZE_1 = "20"
+
+        self.CHECK_CRC_1 = "0000000000000000"
+        self.CHECK_URL_1 = f"http://{mucha_url}/checkUrl/"
+        self.CHECK_SIZE_1 = "20"
+
         self.INFO_SIZE_1 = "0"
         self.COM_SIZE_1 = "0"
         self.COM_TIME_1 = "0"
         self.LAN_INFO_SIZE_1 = "0"
+
         self.USER_ID = ""
         self.PASSWORD = ""
 
+"""
+RESULTS
+EXE_VER
 
+UPDATE_VER_%d
+UPDATE_URL_%d
+UPDATE_SIZE_%d
+
+CHECK_CRC_%d
+CHECK_URL_%d
+CHECK_SIZE_%d
+
+INFO_SIZE_1
+COM_SIZE_1
+COM_TIME_1
+LAN_INFO_SIZE_1
+
+USER_ID
+PASSWORD
+"""
 class MuchaUpdateResponseStub:
     def __init__(self, game_ver: str) -> None:
         self.RESULTS = "001"
         self.UPDATE_VER_1 = game_ver
+
+class MuchaDownloadStateRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.updateVer = request.get("updateVer", "")
+        self.serialNum = request.get("serialNum", "")
+        self.fileSize = request.get("fileSize", "")
+        self.compFileSize = request.get("compFileSize", "")
+        self.boardId = request.get("boardId", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaDownloadErrorRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.updateVer = request.get("updateVer", "")
+        self.serialNum = request.get("serialNum", "")
+        self.downloadUrl = request.get("downloadUrl", "")
+        self.errCd = request.get("errCd", "")
+        self.errMessage = request.get("errMessage", "")
+        self.boardId = request.get("boardId", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaRegiAuthRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.serialNum = request.get("serialNum", "") # Encrypted
+        self.countryCd = request.get("countryCd", "")
+        self.registrationCd = request.get("registrationCd", "")
+        self.sendDate = request.get("sendDate", "")
+        self.useToken = request.get("useToken", "")
+        self.allToken = request.get("allToken", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaRegiAuthResponse:
+    def __init__(self) -> None:
+        self.RESULTS = "001" # 001 = success, 099, 098, 097 = fail, others = fail
+        self.ALL_TOKEN = "0" # Encrypted
+        self.ADD_TOKEN = "0" # Encrypted
+
+class MuchaTokenStateRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.serialNum = request.get("serialNum", "")
+        self.countryCd = request.get("countryCd", "")
+        self.useToken = request.get("useToken", "")
+        self.allToken = request.get("allToken", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+class MuchaTokenStateResponse:
+    def __init__(self) -> None:
+        self.RESULTS = "001"
+
+class MuchaTokenMarginStateRequest:
+    def __init__(self, request: Dict) -> None:        
+        self.gameCd = request.get("gameCd", "")
+        self.serialNum = request.get("serialNum", "")
+        self.countryCd = request.get("countryCd", "")
+        self.placeId = request.get("placeId", "")
+        self.limitLowerToken = request.get("limitLowerToken", 0)
+        self.limitUpperToken = request.get("limitUpperToken", 0)
+        self.settlementMonth = request.get("settlementMonth", 0)
+
+class MuchaTokenMarginStateResponse:
+    def __init__(self) -> None:
+        self.RESULTS = "001"
+        self.LIMIT_LOWER_TOKEN = 0
+        self.LIMIT_UPPER_TOKEN = 0
+        self.LAST_SETTLEMENT_MONTH = 0
+        self.LAST_LIMIT_LOWER_TOKEN = 0
+        self.LAST_LIMIT_UPPER_TOKEN = 0
+        self.SETTLEMENT_MONTH = 0

core/title.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any
+from typing import Dict, List, Tuple
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 from twisted.web.http import Request
@@ -7,14 +7,88 @@ from core.config import CoreConfig
 from core.data import Data
 from core.utils import Utils
 
+class BaseServlet:
+    def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
+        self.core_cfg = core_cfg
+        self.game_cfg = None
+        self.logger = logging.getLogger("title")
+
+    @classmethod
+    def is_game_enabled(cls, game_code: str, core_cfg: CoreConfig, cfg_dir: str) -> bool:
+        """Called during boot to check if a specific game code should load.
+
+        Args:
+            game_code (str): 4 character game code
+            core_cfg (CoreConfig): CoreConfig class
+            cfg_dir (str): Config directory
+
+        Returns:
+            bool: True if the game is enabled and set to run, False otherwise
+        
+        """
+        return False
+    
+    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
+        """Called during boot to get all matcher endpoints this title servlet handles
+
+        Returns:
+            Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]: A 2-length tuple where offset 0 is GET and offset 1 is POST, 
+            containing a list of 3-length tuples where offset 0 is the name of the function in the handler that should be called, offset 1
+            is the matching string, and offset 2 is a dict containing rules for the matcher.
+        """
+        return (
+            [("render_GET", "/{game}/{version}/{endpoint}", {'game': R'S...'})], 
+            [("render_POST", "/{game}/{version}/{endpoint}", {'game': R'S...'})]
+        )
+    
+    def setup(self) -> None:
+        """Called once during boot, should contain any additional setup the handler must do, such as starting any sub-services
+        """
+        pass
+
+    def get_allnet_info(self, game_code: str, game_ver: int, keychip: str) -> Tuple[str, str]:
+        """Called any time a request to PowerOn is made to retrieve the url/host strings to be sent back to the game
+
+        Args:
+            game_code (str): 4 character game code
+            game_ver (int): version, expressed as an integer by multiplying by 100 (1.10 -> 110)
+            keychip (str): Keychip serial of the requesting machine, can be used to deliver specific URIs to different machines
+
+        Returns:
+            Tuple[str, str]: A tuple where offset 0 is the allnet uri field, and offset 1 is the allnet host field
+        """
+        if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
+            return (f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", "")
+
+        return (f"http://{self.core_cfg.title.hostname}/{game_code}/{game_ver}/", "")
+
+    def get_mucha_info(self, core_cfg: CoreConfig, cfg_dir: str) -> Tuple[bool, str]:
+        """Called once during boot to check if this game is a mucha game
+
+        Args:
+            core_cfg (CoreConfig): CoreConfig class
+            cfg_dir (str): Config directory
+
+        Returns:
+            Tuple[bool, str]: Tuple where offset 0 is true if the game is enabled, false otherwise, and offset 1 is the game CD
+        """
+        return (False, "")
+
+    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+        self.logger.warn(f"{game_code} Does not dispatch POST")
+        return None
+
+    def render_GET(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+        self.logger.warn(f"{game_code} Does not dispatch GET")
+        return None
 
 class TitleServlet:
+    title_registry: Dict[str, BaseServlet] = {}
     def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
         super().__init__()
         self.config = core_cfg
         self.config_folder = cfg_folder
         self.data = Data(core_cfg)
-        self.title_registry: Dict[str, Any] = {}
 
         self.logger = logging.getLogger("title")
         if not hasattr(self.logger, "initialized"):
@@ -43,62 +117,62 @@ class TitleServlet:
         plugins = Utils.get_all_titles()
 
         for folder, mod in plugins.items():
-            if hasattr(mod, "game_codes") and hasattr(mod, "index"):
+            if hasattr(mod, "game_codes") and hasattr(mod, "index") and hasattr(mod.index, "is_game_enabled"):
                 should_call_setup = True
+                game_servlet: BaseServlet = mod.index
+                game_codes: List[str] = mod.game_codes
+                
+                for code in game_codes:
+                    if game_servlet.is_game_enabled(code, self.config, self.config_folder):
+                        handler_cls = game_servlet(self.config, self.config_folder)
 
-                if hasattr(mod.index, "get_allnet_info"):
-                    for code in mod.game_codes:
-                        enabled, _, _ = mod.index.get_allnet_info(
-                            code, self.config, self.config_folder
-                        )
+                        if hasattr(handler_cls, "setup") and should_call_setup:
+                            handler_cls.setup()
+                            should_call_setup = False
 
-                        if enabled:
-                            handler_cls = mod.index(self.config, self.config_folder)
-
-                            if hasattr(handler_cls, "setup") and should_call_setup:
-                                handler_cls.setup()
-                                should_call_setup = False
-
-                            self.title_registry[code] = handler_cls
-
-                else:
-                    self.logger.warn(f"Game {folder} has no get_allnet_info")
+                        self.title_registry[code] = handler_cls
 
             else:
-                self.logger.error(f"{folder} missing game_code or index in __init__.py")
+                self.logger.error(f"{folder} missing game_code or index in __init__.py, or is_game_enabled in index")
 
         self.logger.info(
             f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.title.port) if core_cfg.title.port > 0 else ''}"
         )
 
     def render_GET(self, request: Request, endpoints: dict) -> bytes:
-        code = endpoints["game"]
+        code = endpoints["title"]
+        subaction = endpoints['subaction']
+        
         if code not in self.title_registry:
-            self.logger.warn(f"Unknown game code {code}")
+            self.logger.warning(f"Unknown game code {code}")
             request.setResponseCode(404)
             return b""
 
         index = self.title_registry[code]
-        if not hasattr(index, "render_GET"):
-            self.logger.warn(f"{code} does not dispatch GET")
-            request.setResponseCode(405)
+        handler = getattr(index, f"{subaction}", None)
+        if handler is None:
+            self.logger.error(f"{code} does not have handler for GET subaction {subaction}")
+            request.setResponseCode(500)
             return b""
 
-        return index.render_GET(request, endpoints["version"], endpoints["endpoint"])
+        return handler(request, code, endpoints)
 
     def render_POST(self, request: Request, endpoints: dict) -> bytes:
-        code = endpoints["game"]
+        code = endpoints["title"]
+        subaction = endpoints['subaction']
+
         if code not in self.title_registry:
-            self.logger.warn(f"Unknown game code {code}")
+            self.logger.warning(f"Unknown game code {code}")
             request.setResponseCode(404)
             return b""
 
         index = self.title_registry[code]
-        if not hasattr(index, "render_POST"):
-            self.logger.warn(f"{code} does not dispatch POST")
-            request.setResponseCode(405)
+        handler = getattr(index, f"{subaction}", None)
+        if handler is None:
+            self.logger.error(f"{code} does not have handler for POST subaction {subaction}")
+            request.setResponseCode(500)
             return b""
 
-        return index.render_POST(
-            request, int(endpoints["version"]), endpoints["endpoint"]
-        )
+        endpoints.pop("title")
+        endpoints.pop("subaction")
+        return handler(request, code, endpoints)

core/utils.py

@@ -1,12 +1,18 @@
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 from types import ModuleType
 from twisted.web.http import Request
 import logging
 import importlib
 from os import walk
+import jwt
+from base64 import b64decode
+from datetime import datetime, timezone
 
+from .config import CoreConfig
 
 class Utils:
+    real_title_port = None
+    real_title_port_ssl = None
     @classmethod
     def get_all_titles(cls) -> Dict[str, ModuleType]:
         ret: Dict[str, Any] = {}
@@ -33,3 +39,59 @@ class Utils:
             if b"x-forwarded-for" in req.getAllHeaders()
             else req.getClientAddress().host
         )
+    
+    @classmethod
+    def get_title_port(cls, cfg: CoreConfig):
+        if cls.real_title_port is not None: return cls.real_title_port
+
+        if cfg.title.port == 0:
+            cls.real_title_port = cfg.allnet.port
+        
+        else:
+            cls.real_title_port = cfg.title.port
+        
+        return cls.real_title_port
+
+    @classmethod
+    def get_title_port_ssl(cls, cfg: CoreConfig):
+        if cls.real_title_port_ssl is not None: return cls.real_title_port_ssl
+
+        if cfg.title.port_ssl == 0:
+            cls.real_title_port_ssl = 443
+        
+        else:
+            cls.real_title_port_ssl = cfg.title.port_ssl
+        
+        return cls.real_title_port_ssl
+
+def create_sega_auth_key(aime_id: int, game: str, place_id: int, keychip_id: str, b64_secret: str, exp_seconds: int = 86400, err_logger: str = 'aimedb') -> Optional[str]:
+    logger = logging.getLogger(err_logger)
+    try:
+        return jwt.encode({ "aime_id": aime_id, "game": game, "place_id": place_id, "keychip_id": keychip_id, "exp": int(datetime.now(tz=timezone.utc).timestamp()) + exp_seconds }, b64decode(b64_secret), algorithm="HS256")
+    except jwt.InvalidKeyError:
+        logger.error("Failed to encode Sega Auth Key because the secret is invalid!")
+        return None
+    except Exception as e:
+        logger.error(f"Unknown exception occoured when encoding Sega Auth Key! {e}")
+        return None
+
+def decode_sega_auth_key(token: str, b64_secret: str, err_logger: str = 'aimedb') -> Optional[Dict]:
+    logger = logging.getLogger(err_logger)
+    try:
+        return jwt.decode(token, "secret", b64decode(b64_secret), algorithms=["HS256"], options={"verify_signature": True})
+    except jwt.ExpiredSignatureError:
+        logger.error("Sega Auth Key failed to validate due to an expired signature!")
+        return None
+    except jwt.InvalidSignatureError:
+        logger.error("Sega Auth Key failed to validate due to an invalid signature!")
+        return None
+    except jwt.DecodeError as e:
+        logger.error(f"Sega Auth Key failed to decode! {e}")
+        return None
+    except jwt.InvalidTokenError as e:
+        logger.error(f"Sega Auth Key is invalid! {e}")
+        return None
+    except Exception as e:
+        logger.error(f"Unknown exception occoured when decoding Sega Auth Key! {e}")
+        return None
+    

dbutils.py

@@ -1,3 +1,5 @@
+import os
+
 import yaml
 import argparse
 import logging
@@ -56,10 +58,10 @@ if __name__ == "__main__":
 
     elif args.action == "upgrade" or args.action == "rollback":
         if args.version is None:
-            data.logger.warn("No version set, upgrading to latest")
+            data.logger.warning("No version set, upgrading to latest")
 
         if args.game is None:
-            data.logger.warn("No game set, upgrading core schema")
+            data.logger.warning("No game set, upgrading core schema")
             data.migrate_database(
                 "CORE",
                 int(args.version) if args.version is not None else None,
@@ -84,5 +86,11 @@ if __name__ == "__main__":
 
     elif args.action == "cleanup":
         data.delete_hanging_users()
+    
+    elif args.action == "version":
+        data.show_versions()
+
+    elif args.action == "alembic_upgrade":
+        data.alembic_upgrade(os.path.join(os.path.dirname(__file__), 'alembic.ini'))
 
     data.logger.info("Done")

docker-compose.yml

@@ -0,0 +1,66 @@
+version: "3.9"
+services:
+  app:
+    hostname: ma.app
+    build: .
+    volumes:
+      - ./aime:/app/aime
+      - ./configs/config:/app/config
+
+    environment:
+      CFG_DEV: 1
+      CFG_CORE_SERVER_HOSTNAME: 0.0.0.0
+      CFG_CORE_DATABASE_HOST: ma.db
+      CFG_CORE_MEMCACHED_HOSTNAME: ma.memcached
+      CFG_CORE_AIMEDB_KEY: <INSERT AIMEDB KEY HERE>
+      CFG_CHUNI_SERVER_LOGLEVEL: debug
+
+    ports:
+      - "80:80"
+      - "8443:8443"
+      - "22345:22345"
+
+      - "8080:8080"
+      - "8090:8090"
+
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    hostname: ma.db
+    image: yobasystems/alpine-mariadb:10.11.5
+    environment:
+      MYSQL_DATABASE: aime
+      MYSQL_USER: aime
+      MYSQL_PASSWORD: aime
+      MYSQL_ROOT_PASSWORD: AimeRootPassword
+      MYSQL_CHARSET: utf8mb4
+      MYSQL_COLLATION: utf8mb4_general_ci
+    ##Note: expose port 3306 to allow read.py importer into database, comment out when not needed
+    #ports:
+    #  - "3306:3306"
+    ##Note: uncomment to allow mysql to create a persistent database, leave commented if you want to rebuild database from scratch often
+    #volumes:
+    #  - ./AimeDB:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost", "-pAimeRootPassword"]
+      timeout: 5s
+      retries: 5
+
+  memcached:
+    hostname: ma.memcached
+    image: memcached:1.6.22-alpine3.18
+    command: [ "memcached", "-m", "1024", "-I", "128m" ]
+
+  phpmyadmin:
+    hostname: ma.phpmyadmin
+    image: phpmyadmin:latest
+    environment:
+      PMA_HOSTS: ma.db
+      PMA_USER: root
+      PMA_PASSWORD: AimeRootPassword
+      APACHE_PORT: 8080
+    ports:
+    - "9090:8080"
+

docs/INSTALL_DOCKER.md

@@ -0,0 +1,246 @@
+# ARTEMiS - Docker Installation Guide
+
+This step-by-step guide will allow you to install a Contenerized Version of ARTEMiS inside Docker, some steps can be skipped assuming you already have pre-requisite components and modules installed.
+
+This guide assumes using Debian 12(bookworm-stable) as a Host Operating System for most of packages and modules.
+
+## Pre-Requisites:
+
+- Linux-Based Operating System (e.g. Debian, Ubuntu)
+- Docker (https://get.docker.com)
+- Python 3.9+
+- (optional) Git
+
+## Install Python3.9+ and Docker
+
+```
+(if this is a fresh install of the system)
+sudo apt update && sudo apt upgrade
+
+(installs python3 and pip)
+sudo apt install python3 python3-pip
+
+(installs docker)
+curl -fsSL https://get.docker.com -o get-docker.sh
+sh get-docker.sh
+
+(optionally install git)
+sudo apt install git
+```
+
+## Get ARTEMiS
+
+If you installed git, clone into your choice of ARTEMiS git repository, e.g.:
+```
+git clone <ARTEMiS Repo> <folder>
+```
+If not, download the source package, and unpack it to the folder of your choice.
+
+## Prepare development/home configuration
+
+To build our Docker setup, first we need to create some folders and copy some files around
+- Create 'aime', 'configs', 'AimeDB', and 'logs' folder in ARTEMiS root folder (where all source files exist)
+- Inside configs folder, create 'config' folder, and copy all .yaml files from example_config to config (thats all files without nginx_example.conf)
+- Edit .yaml files inside configs/config to suit your server needs
+- Edit core.yaml inside configs/config:
+```
+set server.listen_address: to "0.0.0.0"
+set title.hostname: to machine's IP address, e.g. "192.168.x.x", depending on your network, or actual hostname if your configuration is already set for dns resolve
+set database.host: to "ma.db"
+set database.memcached_host: to "ma.memcached"
+set aimedb.key: to "<actual AIMEDB key>"
+```
+
+## Running Docker Compose
+
+After configuring, go to ARTEMiS root folder, and execute:
+
+```
+docker compose up -d
+```
+
+("-d" argument means detached or daemon, meaning you will regain control of your terminal and Containers will run in background)
+
+This will start pulling and building required images from network, after it's done, a development server should be running, with server accessible under machine's IP, frontend with port 8090, and PHPMyAdmin under port 9090.
+
+- To turn off the server, from ARTEMiS root folder, execute:
+
+```
+docker compose down
+```
+
+- If you changed some files around, and don't see your changes applied, execute:
+
+```
+(turn off the server)
+docker compose down
+(rebuild)
+docker compose build
+(turn on)
+docker compose up -d
+```
+
+- If you need to see logs from containers running, execute:
+
+```
+docker compose logs
+```
+
+- add '-f' to the end if you want to follow logs.
+
+## Running commands
+
+If you need to execute python scripts supplied with the application, use `docker compose exec app python3 <script> <command>`, for example `docker compose exec app python3 dbutils.py version`
+
+## Persistent DB
+
+By default, in development mode, ARTEMiS database is stored temporarily, if you wish to keep your database saved between restarts, we need to bind the database inside the container to actual storage/folder inside our server, to do this we need to make a few changes:
+
+- First off, edit docker-compose.yml, and uncomment 2 lines:
+
+```
+(uncomment these two)
+#volumes:
+#  - ./AimeDB:/var/lib/mysql
+```
+
+- After that, start up the server, this time Database will be saved in AimeDB folder we created in our configuration steps.
+- If you wish to save it in another folder and/or storage device, change the "./AimeDB" target folder to folder/device of your choice
+
+NOTE (NEEDS FIX): at the moment running development mode with persistent DB will always run database creation script at the start of application, while it doesn't break database outright, it might create some issues, a temporary fix can be applied:
+
+- Start up containers with persistent DB already enabled, let application create database
+- After startup, `docker compose down` the instance
+- Edit entrypoint.sh and remove the `python3 dbutils.py create` line from Development mode statement
+- Execute `docker compose build` and `docker compose up -d` to rebuild the app and start the containers back
+
+## Adding importer data
+
+To add data using importer, we can do that a few ways:
+
+### Use importer locally on server
+
+For that we need actual GameData and Options supplied somehow to the server system, be it wsl2 mounting layer, a pendrive with data, network share, or a direct copy to the server storage
+With python3 installed on system, install requirements.txt directly to the system, or through python3 virtual-environment (python3-venv)
+Default mysql/mariadb client development packages will also be required
+
+- In the system:
+
+```
+sudo apt install default-libmysqlclient-dev build-essential pkg-config libmemcached-dev
+sudo apt install mysql-client
+OR
+sudo apt install libmariadb-dev
+```
+
+- In the root ARTEMiS folder
+
+```
+python3 -m pip install -r requirements.txt
+```
+
+- If we wish to layer that with python3 virtual-environment, install required system packages, then:
+
+```
+sudo apt install python3-venv
+python3 -m venv /path/to/venv
+cd /path/to/venv/bin
+python3 -m pip install -r /path/to/artemis/requirements.txt
+```
+
+- Depending on how you installed, now you can run read.py using:
+- For direct installation, from root ARTEMiS folder:
+
+```
+python3 read.py <args>
+```
+
+- Or from python3 virtual environment, from root ARTEMiS folder:
+
+```
+/path/to/python3-venv/bin/python3 /path/to/artemis/read.py <args>
+```
+
+- We need to expose database container port, so that read.py can communicate with the database, inside docker-compose.yml, uncomment 2 lines in the database container declaration (db):
+
+```
+#ports:
+#  - "3306:3306"
+```
+
+- Now, `docker compose down && docker compose build && docker compose up -d` to restart containers
+
+Now to insert the data, by default, docker doesn't expose container hostnames to root system, when trying to run read.py against a container, it will Error that hostname is not available, to fix that, we can add database hostname by hand to /etc/hosts:
+
+```
+sudo <editor of your choice> /etc/hosts
+add '127.0.0.1  ma.db' to the table
+save and close
+```
+
+- You can remove the line in /etc/hosts and de-expose the database port after successful import (this assumes you're using Persistent DB, as restarting the container without it will clear imported data).
+
+### Use importer on remote Linux system
+
+Follow the system and python portion of the guide, installing required packages and python3 modules, Download the ARTEMiS source.
+
+- Edit core.yaml and insert it into config catalog:
+
+```
+database:
+  host: "<hostname of target system>"
+```
+
+- Expose port 3306 from database docker container to system, and allow port 3306 through system firewall to expose port to the system from which you will be importing data. (Remember to close down the database ports after finishing!)
+
+- Import data using read.py
+
+### Use importer on remote Windows system
+
+Follow the [windows](docs/INSTALL_WINDOWS.md) guide for installing python dependencies, download the ARTEMiS source.
+
+- Edit core.yaml and insert it into config catalog:
+
+```
+database:
+  host: "<hostname of target system>"
+```
+
+- Expose port 3306 from database docker container to system, and allow port 3306 through system firewall to expose port to the system from which you will be importing data.
+- For Windows, also allow port 3306 outside the system so that read.py can communicate with remote database. (Remember to close down the database ports after finishing!)
+
+# Troubleshooting
+
+## Game does not connect to ARTEMiS Allnet Server
+Double check your core.yaml if all addresses are correct and ports are correctly set and/or opened.
+
+## Game does not connect to Title Server
+Title server hostname requires your actual system hostname, from which you set up the Containers, or it's IP address, you can get the IP by using command `ip a` which will list all interfaces, and one of them should be your system IP (typically under eth0).
+
+## Unhandled command in AimeDB
+Make sure you have a proper AimeDB Key added to configuration.
+
+## Memcached Error in ARTEMiS application causes errors in loading data
+Currently when running ARTEMiS from master branch, there is a small bug that causes app to always configure memcached service to 127.0.0.1, to fix that, locate cache.py file in core/data, and edit:
+
+```
+memcache = pylibmc.Client([hostname]), binary=True)
+```
+
+to:
+
+```
+memcache = pylibmc.Client(["ma.memcached"], binary=True)
+```
+
+And build the containers again.
+This will fix errors loading data from server.
+(This is fixed in development branch)
+
+## read.py "Can't connect to local server through socket '/run/mysqld/mysqld.sock'"
+
+sqlalchemy by default reads any ip based connection as socket, thus trying to connect locally, please use a hostname (such as ma.db as in guide, and do not localhost) to force it to use a network interface.
+
+### TODO:
+
+- Production environment

docs/INSTALL_WINDOWS.md

@@ -10,11 +10,11 @@ This step-by-step guide assumes that you are using a fresh install of Windows 10
     3. Make sure that you enable "Create shortcuts for installed applications" and "Add Python to environment variables" and hit Install
 
 ## Install MySQL 8.0
-1. Download MySQL 8.0 Server : [Link](https://cdn.mysql.com//Downloads/MySQLInstaller/mysql-installer-web-community-8.0.31.0.msi)
-2. Install mysql-installer-web-community-8.0.31.0.msi
+1. Download MySQL 8.0 Server : [Link](https://dev.mysql.com/get/Downloads/MySQLInstaller/mysql-installer-community-8.0.34.0.msi)
+2. Install mysql-installer-web-community-8.0.34.0.msi
     1. Click on "Add ..." on the side
     2. Click on the "+" next to MySQL Servers
-    3. Make sure MySQL Server 8.0.29 - X64 is under the products to be installed.
+    3. Make sure MySQL Server 8.0.34 - X64 is under the products to be installed.
     4. Hit Next and Next once installed
     5. Select the configuration type "Development Computer"
     6. Hit Next
@@ -23,9 +23,10 @@ This step-by-step guide assumes that you are using a fresh install of Windows 10
     9. Leave everything under Windows Service as default and hit Next >
     10. Click on Execute and for it to finish and hit Next> and then Finish
 3. Open MySQL 8.0 Command Line Client and login as your root user
-4. Type those commands to create your user and the database
-```
-CREATE USER 'aime'@'localhost' IDENTIFIED BY 'MyStrongPass.';
+4. Change `<Enter Password Here>` to a new password for the user aime, type those commands to create your user and the database
+
+```sql
+CREATE USER 'aime'@'localhost' IDENTIFIED BY '<Enter Password Here>';
 CREATE DATABASE aime;
 GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
 FLUSH PRIVILEGES;
@@ -34,33 +35,50 @@ exit;
 
 ## Install Python modules
 1. Change your work path to the artemis-master folder using 'cd' and install the requirements:
-> pip install -r requirements.txt
 
-## Copy/Rename the folder example_config to config
+```shell
+pip install -r requirements.txt
+```	
 
-## Adjust /config/core.yaml
+## Copy/Rename the folder `example_config` to `config`
 
-1. Make sure to change the server listen_address to be set to your local machine IP (ex.: 192.168.1.xxx) 
+## Adjust `config/core.yaml`
+
+1. Make sure to change the server `hostname` to be set to your local machine IP (ex.: 192.168.xxx.xxx) 
     - In case you want to run this only locally, set the following values:
-```
+
+```yaml
 server:
     listen_address: 0.0.0.0
 title: 
-    hostname: localhost
+    hostname: 192.168.xxx.xxx
+```
+
+1. Adjust the proper MySQL information you created earlier
+```yaml
+database:
+  host: "localhost"
+  username: "aime"
+  password: "<Enter Password Here>"
+  name: "aime"
 ```
-2. Adjust the proper MySQL information you created earlier
 3. Add the AimeDB key at the bottom of the file
 4. If the webui is needed, change the flag from False to True
 
 ## Create the database tables for ARTEMiS
-> python dbutils.py create
+
+```shell
+python dbutils.py create
+```
 
 ## Firewall Adjustements 
 Make sure the following ports are open both on your router and local Windows firewall in case you want to use this for public use (NOT recommended):
 > Port 80 (TCP), 443 (TCP), 8443 (TCP), 22345 (TCP), 8080 (TCP), 8090 (TCP) **webui, 8444 (TCP) **mucha
 
 ## Running the ARTEMiS instance
-> python index.py
+```shell
+python index.py
+```
 
 # Troubleshooting
 
@@ -78,6 +96,7 @@ Make sure the following ports are open both on your router and local Windows fir
 ## AttributeError: module 'collections' has no attribute 'Hashable'
 1. This means the pyYAML module is obsolete, simply rerun pip with the -U (force update) flag, as shown below.
     - Change your work path to the artemis-master (or artemis-develop) folder using 'cd' and run the following commands:
-```
+
+```shell
 pip install -r requirements.txt -U
 ```

docs/config.md

@@ -5,11 +5,19 @@
 - `allow_unregistered_serials`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
 - `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
 - `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
+- `threading`: Flags that `reactor.run` should be called via the `Thread` standard library. May provide a speed boost, but removes the ability to kill the server via `Ctrl + C`. Default: `False`
+- `check_arcade_ip`: Checks IPs against the `arcade` table in the database, if one is defined. Default `False`
+- `strict_ip_checking`: Rejects clients if there is no IP in the `arcade` table for the respective arcade
 - `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
 ## Title
 - `loglevel`: Logging level for the title server. Default `info`
 - `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
 - `port`: Port that the title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `8080`
+- `port_ssl`: Port that the secure title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `0`
+- `ssl_key`: Location of the ssl server key for the secure title server. Ignored if `port_ssl` is set to `0` or `is_develop` set to `False`. Default `cert/title.key`
+- `ssl_cert`: Location of the ssl server certificate for the secure title server. Must not be a self-signed SSL. Ignored if `port_ssl` is set to `0` or `is_develop` is set to `False`. Default `cert/title.pem`
+- `reboot_start_time`: 24 hour JST time that clients will see as the start of maintenance period. Leave blank for no maintenance time. Default: ""
+- `reboot_end_time`: 24 hour JST time that clients will see as the end of maintenance period. Leave blank for no maintenance time. Default: ""
 ## Database
 - `host`: Host of the database. Default `localhost`
 - `username`: Username of the account the server should connect to the database with. Default `aime`

docs/game_specific_info.md

@@ -6,45 +6,55 @@ the corresponding importer and database upgrades.
 **Important: The described database upgrades are only required if you are using an old database schema, f.e. still
 using the megaime database. Clean installations always create the latest database structure!**
 
+To upgrade the core database and the database for every game, execute:
+
+```shell
+python dbutils.py autoupgrade
+```
+
 # Table of content
 
 - [Supported Games](#supported-games)
-    - [Chunithm](#chunithm)
+    - [CHUNITHM](#chunithm)
     - [crossbeats REV.](#crossbeats-rev)
     - [maimai DX](#maimai-dx)
     - [O.N.G.E.K.I.](#o-n-g-e-k-i)
     - [Card Maker](#card-maker)
     - [WACCA](#wacca)
+    - [Sword Art Online Arcade](#sao)
+	- [Initial D THE ARCADE](#initial-d-the-arcade)
 
 
 # Supported Games
 
 Games listed below have been tested and confirmed working.
 
-## Chunithm
+## CHUNITHM
 
 ### SDBT
 
-| Version ID | Version Name       |
-|------------|--------------------|
-| 0          | Chunithm           |
-| 1          | Chunithm+          |
-| 2          | Chunithm Air       |
-| 3          | Chunithm Air +     |
-| 4          | Chunithm Star      |
-| 5          | Chunithm Star +    |
-| 6          | Chunithm Amazon    |
-| 7          | Chunithm Amazon +  |
-| 8          | Chunithm Crystal   |
-| 9          | Chunithm Crystal + |
-| 10         | Chunithm Paradise  |
+| Version ID | Version Name          |
+| ---------- | --------------------- |
+| 0          | CHUNITHM              |
+| 1          | CHUNITHM PLUS         |
+| 2          | CHUNITHM AIR          |
+| 3          | CHUNITHM AIR PLUS     |
+| 4          | CHUNITHM STAR         |
+| 5          | CHUNITHM STAR PLUS    |
+| 6          | CHUNITHM AMAZON       |
+| 7          | CHUNITHM AMAZON PLUS  |
+| 8          | CHUNITHM CRYSTAL      |
+| 9          | CHUNITHM CRYSTAL PLUS |
+| 10         | CHUNITHM PARADISE     |
 
 ### SDHD/SDBT
 
-| Version ID | Version Name    |
-|------------|-----------------|
-| 11         | Chunithm New!!  |
-| 12         | Chunithm New!!+ |
+| Version ID | Version Name        |
+| ---------- | ------------------- |
+| 11         | CHUNITHM NEW!!      |
+| 12         | CHUNITHM NEW PLUS!! |
+| 13         | CHUNITHM SUN        |
+| 14         | CHUNITHM SUN PLUS   |
 
 
 ### Importer
@@ -52,27 +62,90 @@ Games listed below have been tested and confirmed working.
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SDBT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDBT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
 The importer for Chunithm will import: Events, Music, Charge Items and Avatar Accesories.
 
+### Config
+
+Config file is located in `config/chuni.yaml`.
+
+| Option           | Info                                                                                                           |
+|------------------|----------------------------------------------------------------------------------------------------------------|
+| `news_msg`       | If this is set, the news at the top of the main screen will be displayed (up to Chunithm Paradise Lost)        |
+| `name`           | If this is set, all players that are not on a team will use this one by default.                               |
+| `use_login_bonus`| This is used to enable the login bonuses                                                                       |
+| `crypto`         | This option is used to enable the TLS Encryption                                                               |
+
+
+**If you would like to use network encryption, the following will be required underneath but key, iv and hash are required:**
+
+```yaml
+crypto:
+  encrypted_only: False
+  keys:
+    13: ["0000000000000000000000000000000000000000000000000000000000000000", "00000000000000000000000000000000", "0000000000000000"]
+```
+
 ### Database upgrade
 
-Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
-which version is the latest, f.e. `SDBT_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to
-perform all previous updates as well:
+Always make sure your database (tables) are up-to-date:
 
 ```shell
 python dbutils.py --game SDBT upgrade
 ```
 
+### Online Battle
+
+**Only matchmaking (with your imaginary friends) is supported! Online Battle does not (yet?) work!**
+
+The first person to start the Online Battle (now called host) will create a "matching room" with a given `roomId`, after that max 3 other people can join the created room.
+Non used slots during the matchmaking will be filled with CPUs after the timer runs out.
+As soon as a new member will join the room the timer will jump back to 60 secs again.
+Sending those 4 messages to all other users is also working properly.
+In order to use the Online Battle every user needs the same ICF, same rom version and same data version!
+If a room is full a new room will be created if another user starts an Online Battle.
+After a failed Online Battle the room will be deleted. The host is used for the timer countdown, so if the connection failes to the host the timer will stop and could create a "frozen" state.
+
+#### Information/Problems:
+
+- Online Battle uses UDP hole punching and opens port 50201?
+- `reflectorUri` seems related to that?
+- Timer countdown should be handled globally and not by one user
+- Game can freeze or can crash if someone (especially the host) leaves the matchmaking
+
+### Rivals
+
+You can configure up to 4 rivals in Chunithm on a per-user basis. There is no UI to do this currently, so in the database, you can do this:
+```sql
+INSERT INTO aime.chuni_item_favorite (user, version, favId, favKind) VALUES (<user1>, <version>, <user2>, 2);
+INSERT INTO aime.chuni_item_favorite (user, version, favId, favKind) VALUES (<user2>, <version>, <user1>, 2);
+```
+Note that the version **must match**, otherwise song lookup may not work.
+
+### Teams
+
+You can also configure teams for users to be on. There is no UI to do this currently, so in the database, you can do this:
+```sql
+INSERT INTO aime.chuni_profile_team (teamName) VALUES (<teamName>);
+```
+Team names can be regular ASCII, and they will be displayed ingame.
+
+### Favorite songs
+You can set the songs that will be in a user's Favorite Songs category using the following SQL entries:
+```sql
+INSERT INTO aime.chuni_item_favorite (user, version, favId, favKind) VALUES (<user>, <version>, <songId>, 1);
+```
+The songId is based on the actual ID within your version of Chunithm.
+
+
 ## crossbeats REV.
 
 ### SDCA
 
 | Version ID | Version Name                       |
-|------------|------------------------------------|
+| ---------- | ---------------------------------- |
 | 0          | crossbeats REV.                    |
 | 1          | crossbeats REV. SUNRISE            |
 | 2          | crossbeats REV. SUNRISE S2         |
@@ -83,7 +156,7 @@ python dbutils.py --game SDBT upgrade
 In order to use the importer you need to use the provided `Export.csv` file:
 
 ```shell
-python read.py --series SDCA --version <version ID> --binfolder titles/cxb/data
+python read.py --game SDCA --version <version ID> --binfolder titles/cxb/data
 ```
 
 The importer for crossbeats REV. will import Music.
@@ -92,56 +165,76 @@ The importer for crossbeats REV. will import Music.
 
 Config file is located in `config/cxb.yaml`.
 
-| Option                 | Info                                                       |
-|------------------------|------------------------------------------------------------|
-| `hostname`             | Requires a proper `hostname` (not localhost!) to run       |
-| `ssl_enable`           | Enables/Disables the use of the `ssl_cert` and `ssl_key`   |
-| `port`                 | Set your unsecure port number                              |
-| `port_secure`          | Set your secure/SSL port number                            |
-| `ssl_cert`, `ssl_key`  | Enter your SSL certificate (requires not self signed cert) |
+| Option                | Info                                                       |
+| --------------------- | ---------------------------------------------------------- |
+| `hostname`            | Requires a proper `hostname` (not localhost!) to run       |
+| `ssl_enable`          | Enables/Disables the use of the `ssl_cert` and `ssl_key`   |
+| `port`                | Set your unsecure port number                              |
+| `port_secure`         | Set your secure/SSL port number                            |
+| `ssl_cert`, `ssl_key` | Enter your SSL certificate (requires not self signed cert) |
 
 
 ## maimai DX
 
-### SDEZ
+### Versions
 
-| Version ID | Version Name            |
-|------------|-------------------------|
-| 0          | maimai DX               |
-| 1          | maimai DX PLUS          |
-| 2          | maimai DX Splash        |
-| 3          | maimai DX Splash PLUS   |
-| 4          | maimai DX Universe      |
-| 5          | maimai DX Universe PLUS |
-| 6          | maimai DX Festival      |
+| Game Code | Version ID | Version Name            |
+| --------- | ---------- | ----------------------- |
+| SBXL      | 0          | maimai                  |
+| SBXL      | 1          | maimai PLUS             |
+| SBZF      | 2          | maimai GreeN            |
+| SBZF      | 3          | maimai GreeN PLUS       |
+| SDBM      | 4          | maimai ORANGE           |
+| SDBM      | 5          | maimai ORANGE PLUS      |
+| SDCQ      | 6          | maimai PiNK             |
+| SDCQ      | 7          | maimai PiNK PLUS        |
+| SDDK      | 8          | maimai MURASAKi         |
+| SDDK      | 9          | maimai MURASAKi PLUS    |
+| SDDZ      | 10         | maimai MiLK             |
+| SDDZ      | 11         | maimai MiLK PLUS        |
+| SDEY      | 12         | maimai FiNALE           |
+| SDEZ      | 13         | maimai DX               |
+| SDEZ      | 14         | maimai DX PLUS          |
+| SDEZ      | 15         | maimai DX Splash        |
+| SDEZ      | 16         | maimai DX Splash PLUS   |
+| SDEZ      | 17         | maimai DX UNiVERSE      |
+| SDEZ      | 18         | maimai DX UNiVERSE PLUS |
+| SDEZ      | 19         | maimai DX FESTiVAL      |
+| SDEZ      | 20         | maimai DX FESTiVAL PLUS |
 
 ### Importer
 
 In order to use the importer locate your game installation folder and execute:
-
+DX:
 ```shell
-python read.py --series SDEZ --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game <Game Code> --version <Version ID> --binfolder /path/to/StreamingAssets --optfolder /path/to/game/option/folder
+```
+Pre-DX:
+```shell
+python read.py --game <Game Code> --version <Version ID> --binfolder /path/to/data --optfolder /path/to/patch/data
 ```
-
 The importer for maimai DX will import Events, Music and Tickets.
 
-**NOTE: It is required to use the importer because the game will
-crash without Events!**
+The importer for maimai Pre-DX will import Events and Music. Not all games will have patch data. Milk - Finale have file encryption, and need an AES key. That key is not provided by the developers. For games that do use encryption, provide the key, as a hex string, with the `--extra` flag. Ex `--extra 00112233445566778899AABBCCDDEEFF`
+
+**Important: It is required to use the importer because some games may not function properly or even crash without Events!**
 
 ### Database upgrade
 
-Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEZ_2_upgrade.sql`. In order to upgrade to version 2 in this case you need to perform all previous updates as well:
+Always make sure your database (tables) are up-to-date:
 
 ```shell
 python dbutils.py --game SDEZ upgrade
 ```
 
+Pre-Dx uses the same database as DX, so only upgrade using the SDEZ game code!
+
 ## Hatsune Miku Project Diva
 
 ### SBZV
 
 | Version ID | Version Name                    |
-|------------|---------------------------------|
+| ---------- | ------------------------------- |
 | 0          | Project Diva Arcade             |
 | 1          | Project Diva Arcade Future Tone |
 
@@ -151,7 +244,7 @@ python dbutils.py --game SDEZ upgrade
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SBZV --version <version ID> --binfolder /path/to/game/data/diva --optfolder /path/to/game/data/diva/mdata
+python read.py --game SBZV --version <version ID> --binfolder /path/to/game/data/diva --optfolder /path/to/game/data/diva/mdata
 ```
 
 The importer for Project Diva Arcade will all required data in order to use
@@ -162,16 +255,17 @@ the Shop, Modules and Customizations.
 Config file is located in `config/diva.yaml`.
 
 | Option               | Info                                                                                            |
-|----------------------|-------------------------------------------------------------------------------------------------|
+| -------------------- | ----------------------------------------------------------------------------------------------- |
 | `unlock_all_modules` | Unlocks all modules (costumes) by default, if set to `False` all modules need to be purchased   |
 | `unlock_all_items`   | Unlocks all items (customizations) by default, if set to `False` all items need to be purchased |
 
+### Custom PV Lists (databanks)
+
+In order to use custom PV Lists, simply drop in your .dat files inside of /titles/diva/data/ and make sure they are called PvList0.dat, PvList1.dat, PvList2.dat, PvList3.dat and PvList4.dat exactly.
 
 ### Database upgrade
 
-Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
-which version is the latest, f.e. `SBZV_4_upgrade.sql`. In order to upgrade to version 4 in this case you need to
-perform all previous updates as well:
+Always make sure your database (tables) are up-to-date:
 
 ```shell
 python dbutils.py --game SBZV upgrade
@@ -182,15 +276,15 @@ python dbutils.py --game SBZV upgrade
 ### SDDT
 
 | Version ID | Version Name               |
-|------------|----------------------------|
+| ---------- | -------------------------- |
 | 0          | O.N.G.E.K.I.               |
 | 1          | O.N.G.E.K.I. +             |
-| 2          | O.N.G.E.K.I. Summer        |
-| 3          | O.N.G.E.K.I. Summer +      |
-| 4          | O.N.G.E.K.I. Red           |
-| 5          | O.N.G.E.K.I. Red +         |
-| 6          | O.N.G.E.K.I. Bright        |
-| 7          | O.N.G.E.K.I. Bright Memory |
+| 2          | O.N.G.E.K.I. SUMMER        |
+| 3          | O.N.G.E.K.I. SUMMER +      |
+| 4          | O.N.G.E.K.I. R.E.D.        |
+| 5          | O.N.G.E.K.I. R.E.D. +      |
+| 6          | O.N.G.E.K.I. bright        |
+| 7          | O.N.G.E.K.I. bright MEMORY |
 
 
 ### Importer
@@ -198,7 +292,7 @@ python dbutils.py --game SBZV upgrade
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
 The importer for O.N.G.E.K.I. will all all Cards, Music and Events.
@@ -210,42 +304,118 @@ The importer for O.N.G.E.K.I. will all all Cards, Music and Events.
 Config file is located in `config/ongeki.yaml`.
 
 | Option           | Info                                                                                                           |
-|------------------|----------------------------------------------------------------------------------------------------------------|
+| ---------------- | -------------------------------------------------------------------------------------------------------------- |
 | `enabled_gachas` | Enter all gacha IDs for Card Maker to work, other than default may not work due to missing cards added to them |
+| `crypto`         | This option is used to enable the TLS Encryption                                                               |
 
 Note: 1149 and higher are only for Card Maker 1.35 and higher and will be ignored on lower versions.
 
+**If you would like to use network encryption, the following will be required underneath but key, iv and hash are required:**
+
+```yaml
+crypto:
+  encrypted_only: False
+  keys:
+    7: ["0000000000000000000000000000000000000000000000000000000000000000", "00000000000000000000000000000000", "0000000000000000"]
+```
+
 ### Database upgrade
 
-Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
-which version is the latest, f.e. `SDDT_4_upgrade.sql`. In order to upgrade to version 4 in this case you need to
-perform all previous updates as well:
+Always make sure your database (tables) are up-to-date:
 
 ```shell
 python dbutils.py --game SDDT upgrade
 ```
 
+### Controlling Events (Ranking Event, Technical Challenge Event, Mission Event)
+
+Events are controlled by 2 types of enabled events:
+- RankingEvent (type 6), TechChallengeEvent (type 17)
+- AcceptRankingEvent (type 7), AcceptTechChallengeEvent (type 18)
+
+Both Ranking and Accept must be enabled for event to function properly
+
+Event will run for the time specified in startDate and endDate
+
+AcceptRankingEvent and AcceptTechChallengeEvent are reward period for events, which specify from what startDate until endDate you can collect the rewards for attending the event, so the reward period must start in the future, e.g. :
+
+- RankingEvent startDate 2023-12-01 - endDate 2023-12-30 - period in which whole event is running
+- AcceptRankingEvent startDate 2023-12-23 - endDate 2023-12-30 - period in which you can collect rewards for the event
+
+If player misses the AcceptRankingEvent period - ranking will be invalidated and receive lowest reward from the event (typically 500x money)
+
+Technical Challenge Song List:
+
+Songs that are used for Technical Challenge are not stored anywhere in data files, so you need to fill the database table by yourself, you can gather all songs that should be in Technical Challenges from ONGEKI japanese wikis, or, you can create your own sets:
+
+Database table : `ongeki_static_tech_music`
+
+```
+id: Id in table, just increment for each entry
+version: version of the game you want the tech challenge to be in (from RED and up)
+eventId: Id of the event in ongeki_static_events, insert the Id of the TechChallengeEvent (type 17) you want the song be assigned to
+musicId: Id of the song you want to add, use songId from ongeki_static_music table
+level: Difficulty of the song you want to track during the event, from 0(basic) to 3(master)
+
+```
+
+Current implementation of Ranking and Technical Challenge Events are updated on every profile save to the Network, and Ranked on each player login, in official specification, calculation for current rank on the network should be done in the maintenance window
+
+Mission Event (type 13) is a monthly type of event, which is used when another event doesn't have it's own Ranking or Technical Challenge Event running, only one Mission Event should be running at a time, so enable only the specific Mission you want to run currently on the Network
+
+If you're often trying fresh cards, registering new profiles etc., you can also consider disabling all Announcement Events (type 1), as it will disable all the banners that pop up on login (they show up only once though, so if you click through them once they won't show again)
+
+Event type 2 in Database are Advertisement Movies, enable only 1 you want to currently play, and disable others
+
+
+Present and Reward List - populate reward list using read.py
+
+Create present for players by adding an entry in `ongeki_static_present_list`
+```
+id: unique for each entry
+version: game version you want the present be in
+presentId: id of the present - starts with 1001 and go up from that, must be unique for each reward(don't set multiple rewardIds with same presentId)
+presentName: present name which will be shown on the bottom when received
+rewardId: ID of item from ongeki_static_rewards
+stock: how many you want to give (like 5 copies of same card, or 10000 money, etc.)
+message: no idea, can be left empty
+startDate: date when to start giving out
+endDate: date when ends
+```
+
+After inserting present to the table, add the presentId into players `ongeki_static_item`, where itemKind is 9, itemId is the presentId, and stock set 1 and isValid to 1
+
+After that, on next login the present should be received (or whenever it supposed to happen)
+
+
+
 ## Card Maker
 
 ### SDED
 
 | Version ID | Version Name    |
-|------------|-----------------|
-| 0          | Card Maker 1.34 |
+| ---------- | --------------- |
+| 0          | Card Maker 1.30 |
 | 1          | Card Maker 1.35 |
 
 
 ### Support status
 
-* Card Maker 1.34:
-  * Chunithm New!!: Yes
-  * maimai DX Universe: Yes
-  * O.N.G.E.K.I. Bright: Yes
+#### Card Maker 1.30:
+* CHUNITHM NEW!!: Yes
+* maimai DX UNiVERSE: Yes
+* O.N.G.E.K.I. bright: Yes
 
-* Card Maker 1.35:
-  * Chunithm New!!+: Yes
-  * maimai DX Universe PLUS: Yes
-  * O.N.G.E.K.I. Bright Memory: Yes
+#### Card Maker 1.35:
+* CHUNITHM: 
+  * NEW!!: Yes
+  * NEW PLUS!!: Yes (added in A028)
+  * SUN: Yes (added in A032)
+* maimai DX:
+  * UNiVERSE PLUS: Yes
+  * FESTiVAL: Yes (added in A031)
+  * FESTiVAL PLUS: Yes (added in A035)
+* O.N.G.E.K.I. bright MEMORY: Yes
 
 
 ### Importer
@@ -254,28 +424,63 @@ In order to use the importer you need to use the provided `.csv` files (which ar
 option folders:
 
 ```shell
-python read.py --series SDED --version <version ID> --binfolder titles/cm/cm_data --optfolder /path/to/cardmaker/option/folder
+python read.py --game SDED --version <version ID> --binfolder titles/cm/cm_data --optfolder /path/to/cardmaker/option/folder
 ```
 
 **If you haven't already executed the O.N.G.E.K.I. importer, make sure you import all cards!**
 
 ```shell
-python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --game SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
-Also make sure to import all maimai and Chunithm data as well:
+Also make sure to import all maimai DX and CHUNITHM data as well:
 
 ```shell
-python read.py --series SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
+python read.py --game SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
 ```
 
-The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai/Chunithm) and the hardcoded
+The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai DX/CHUNITHM) and the hardcoded
 Cards for each Gacha (O.N.G.E.K.I. only).
 
 **NOTE: Without executing the importer Card Maker WILL NOT work!**
 
 
-### O.N.G.E.K.I. Gachas
+### Config setup
+
+Make sure to update your `config/cardmaker.yaml` with the correct version for each game. To get the current version required to run a specific game, open every opt (Axxx) folder descending until you find all three folders:
+
+- `MU3`: O.N.G.E.K.I.
+- `MAI`: maimai DX
+- `CHU`: CHUNITHM
+
+Inside each folder is a `DataConfig.xml` file, for example:
+
+`MU3/DataConfig.xml`:
+```xml
+  <cardMakerVersion>
+    <major>1</major>
+    <minor>35</minor>
+    <release>3</release>
+  </cardMakerVersion>
+```
+
+Now update your `config/cardmaker.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  1: # Card Maker 1.35
+    ongeki: 1.35.03
+```
+
+For now you also need to update your `config/ongeki.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  7: # O.N.G.E.K.I. bright MEMORY
+    card_maker: 1.35.03
+```
+
+### O.N.G.E.K.I.
 
 Gacha "無料ガチャ" can only pull from the free cards with the following probabilities: 94%: R, 5% SR and 1% chance of
 getting an SSR card
@@ -288,26 +493,30 @@ and 3% chance of getting an SSR card
 All other (limited) gachas can pull from every card added to ongeki_static_cards but with the promoted cards
 (click on the green button under the banner) having a 10 times higher chance to get pulled
 
-### Chunithm Gachas
+### CHUNITHM
 
-All cards in Chunithm (basically just the characters) have the same rarity to it just pulls randomly from all cards
+All cards in CHUNITHM (basically just the characters) have the same rarity to it just pulls randomly from all cards
 from a given gacha but made sure you cannot pull the same card twice in the same 5 times gacha roll.
 
+### maimai DX
+
+Printed maimai DX cards: Freedom (`cardTypeId=6`) or Gold Pass (`cardTypeId=4`) can now be selected during the login process. You can only have ONE Freedom and ONE Gold Pass active at a given time. The cards will expire after 15 days.
+
+Thanks GetzeAvenue for the `selectedCardList` rarity hint!
+
 ### Notes
 
-Card Maker 1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35 will only load an O.N.G.E.K.I.
+Card Maker 1.30-1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35+ will only load an O.N.G.E.K.I.
 Bright Memory profile (1.35).
-The gachas inside the `ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
+The gachas inside the `config/ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
 Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded for CM 1.35.
 
-**NOTE: There is currently no way to load/use the (printed) maimai DX cards!**
-
 ## WACCA
 
 ### SDFE
 
 | Version ID | Version Name  |
-|------------|---------------|
+| ---------- | ------------- |
 | 0          | WACCA         |
 | 1          | WACCA S       |
 | 2          | WACCA Lily    |
@@ -320,7 +529,7 @@ Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded fo
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --series SDFE --version <version ID> --binfolder /path/to/game/WindowsNoEditor/Mercury/Content
+python read.py --game SDFE --version <version ID> --binfolder /path/to/game/WindowsNoEditor/Mercury/Content
 ```
 
 The importer for WACCA will import all Music data.
@@ -330,7 +539,7 @@ The importer for WACCA will import all Music data.
 Config file is located in `config/wacca.yaml`.
 
 | Option             | Info                                                                        |
-|--------------------|-----------------------------------------------------------------------------|
+| ------------------ | --------------------------------------------------------------------------- |
 | `always_vip`       | Enables/Disables VIP, if disabled it needs to be purchased manually in game |
 | `infinite_tickets` | Always set the "unlock expert" tickets to 5                                 |
 | `infinite_wp`      | Sets the user WP to `999999`                                                |
@@ -339,8 +548,227 @@ Config file is located in `config/wacca.yaml`.
 
 ### Database upgrade
 
-Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDFE_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
+Always make sure your database (tables) are up-to-date:
 
 ```shell
 python dbutils.py --game SDFE upgrade
 ```
+
+### VIP Rewards
+Below is a list of VIP rewards. Currently, VIP is not implemented, and thus these are not obtainable. These 23 rewards were distributed once per month for VIP users on the real network.
+
+	Plates:
+		211004 リッチ
+		211018 特盛えりざべす
+		211025 イースター
+		211026 特盛りりぃ
+		311004 ファンシー
+		311005 インカンテーション
+		311014 夜明け
+		311015 ネイビー
+		311016 特盛るーん
+	
+	Ring Colors:
+		203002 Gold Rushイエロー
+		203009 トロピカル
+		303005 ネイチャー
+	
+	Icons:
+		202020 どらみんぐ
+		202063 ユニコーン
+		202086 ゴリラ
+		302014 ローズ
+		302015 ファラオ
+		302045 肉球
+		302046 WACCA
+		302047 WACCA Lily
+		302048 WACCA Reverse
+	
+	Note Sound Effect:
+		205002 テニス
+		205008 シャワー
+		305003 タンバリンMk-Ⅱ
+
+## SAO
+
+### SDEW
+
+| Version ID | Version Name |
+| ---------- | ------------ |
+| 0          | SAO          |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --game SDEW --version <version ID> --binfolder /path/to/game/extractedassets
+```
+
+The importer for SAO will import all items, heroes, support skills and titles data.
+
+### Config
+
+Config file is located in `config/sao.yaml`.
+
+| Option          | Info                                                              |
+| --------------- | ----------------------------------------------------------------- |
+| `hostname`      | Changes the server listening address for Mucha                    |
+| `port`          | Changes the listing port                                          |
+| `auto_register` | Allows the game to handle the automatic registration of new cards |
+
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date:
+
+```shell
+python dbutils.py --game SDEW upgrade
+```
+
+### Notes
+- Defrag Match will crash at loading
+- Co-Op Online is not supported
+- Shop is not functionnal
+- Player title is currently static and cannot be changed in-game
+- QR Card Scanning currently only load a static hero
+
+**Network hashing in GssSite.dll must be disabled**
+
+### Credits for SAO support:
+
+- Midorica - Limited Network Support
+- Dniel97 - Helping with network base
+- tungnotpunk - Source
+
+## Initial D THE ARCADE
+
+### SDGT
+
+| Version ID | Version Name                  |
+| ---------- | ----------------------------- |
+| 0          | Initial D THE ARCADE Season 1 |
+| 1          | Initial D THE ARCADE Season 2 |
+
+**Important: Only version 1.50.00 (Season 2) is currently working and actively supported!**
+
+### Profile Importer
+
+In order to use the profile importer download the `idac_profile.json` file from the frontend
+and either directly use the folder path with `idac_profile.json` in it or specify the complete
+path to the `.json` file
+
+```shell
+python read.py --game SDGT --version <Version ID> --optfolder /path/to/game/download/folder
+```
+
+The importer for SDGT will import the complete profile data with personal high scores as well.
+
+### Config
+
+Config file is located in `config/idac.yaml`.
+
+| Option                        | Info                                                                                                        |
+| ----------------------------- | ----------------------------------------------------------------------------------------------------------- |
+| `ssl`                         | Enables/Disables the use of the `ssl_cert` and `ssl_key` (currently unsuported)                             |
+| `matching_host`               | IPv4 address of your PC for the Online Battle (currently unsupported)                                       |
+| `port_matching`               | Port number for the Online Battle Matching                                                                  |
+| `port_echo1/2`                | Port numbers for Echos                                                                                      |
+| `port_matching_p2p`           | Port number for Online Battle (currently unsupported)                                                       |
+| `stamp.enable`               | Enables/Disabled the play stamp events                                                                      |
+| `stamp.enabled_stamps`        | Define up to 3 play stamp events (without `.json` extension, which are placed in `titles/idac/data/stamps`) |
+| `timetrial.enable`           | Enables/Disables the time trial event                                                                       |
+| `timetrial.enabled_timetrial` | Define one! trial event (without `.json` extension, which are placed in `titles/idac/data/timetrial`)       |
+
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date:
+
+```shell
+python dbutils.py --game SDGT upgrade
+```
+
+### Notes
+- Online Battle is not supported
+- Online Battle Matching is not supported
+
+### Item categories
+
+| Category ID | Category Name            |
+| ----------- | ------------------------ |
+| 1           | D Coin                   |
+| 3           | Car Dressup Token        |
+| 5           | Avatar Dressup Token     |
+| 6           | Tachometer               |
+| 7           | Aura                     |
+| 8           | Aura Color               |
+| 9           | Avatar Face              |
+| 10          | Avatar Eye               |
+| 11          | Avatar Mouth             |
+| 12          | Avatar Hair              |
+| 13          | Avatar Glasses           |
+| 14          | Avatar Face accessories  |
+| 15          | Avatar Body              |
+| 18          | Avatar Background        |
+| 21          | Chat Stamp               |
+| 22          | Keychain                 |
+| 24          | Title                    |
+| 25          | FullTune Ticket          |
+| 26          | Paper Cup                |
+| 27          | BGM                      |
+| 28          | Drifting Text            |
+| 31          | Start Menu BG            |
+| 32          | Car Color/Paint          |
+| 33          | Aura Level               |
+| 34          | FullTune Ticket Fragment |
+| 35          | Underneon Lights         |
+
+### TimeRelease Chapter:
+
+1. Story: 1, 2, 3, 4, 5, 6, 7, 8, 9, 19 (Chapter 10), (29 Chapter 11?)
+2. MF Ghost: 10, 11, 12, 13, 14, 15
+3. Bunta: 15, 16, 17, 18, 19, 20, (21, 21, 22?)
+4. Special Event: 23, 24, 25, 26, 27, 28 (Touhou Project)
+
+### TimeRelease Courses:
+
+
+| Course ID | Course Name               | Direction                |
+| --------- | ------------------------- | ------------------------ |
+| 0         | Akina Lake(秋名湖)        | CounterClockwise(左周り) |
+| 2         | Akina Lake(秋名湖)        | Clockwise(右周り)        |
+| 52        | Hakone(箱根)              | Downhill(下り)           |
+| 54        | Hakone(箱根)              | Hillclimb(上り)          |
+| 36        | Usui(碓氷)                | CounterClockwise(左周り) |
+| 38        | Usui(碓氷)                | Clockwise(右周り)        |
+| 4         | Myogi(妙義)               | Downhill(下り)           |
+| 6         | Myogi(妙義)               | Hillclimb(上り)          |
+| 8         | Akagi(赤城)               | Downhill(下り)           |
+| 10        | Akagi(赤城)               | Hillclimb(上り)          |
+| 12        | Akina(秋名)               | Downhill(下り)           |
+| 14        | Akina(秋名)               | Hillclimb(上り)          |
+| 16        | Irohazaka(いろは坂)       | Downhill(下り)           |
+| 18        | Irohazaka(いろは坂)       | Reverse(逆走)            |
+| 56        | Momiji Line(もみじライン) | Downhill(下り)           |
+| 58        | Momiji Line(もみじライン) | Hillclimb(上り)          |
+| 20        | Tsukuba(筑波)             | Outbound(往路)           |
+| 22        | Tsukuba(筑波)             | Inbound(復路)            |
+| 24        | Happogahara(八方ヶ原)     | Outbound(往路)           |
+| 26        | Happogahara(八方ヶ原)     | Inbound(復路)            |
+| 40        | Sadamine(定峰)            | Downhill(下り)           |
+| 42        | Sadamine(定峰)            | Hillclimb(上り)          |
+| 44        | Tsuchisaka(土坂)          | Outbound(往路)           |
+| 46        | Tsuchisaka(土坂)          | Inbound(復路)            |
+| 48        | Akina Snow(秋名雪)        | Downhill(下り)           |
+| 50        | Akina Snow(秋名雪)        | Hillclimb(上り)          |
+| 68        | Odawara(小田原)           | Forward(順走)            |
+| 70        | Odawara(小田原)           | Reverse(逆走)            |
+
+### Credits
+- Bottersnike: For the HUGE Reverse Engineering help
+- Kinako: For helping with the timeRelease unlocking of courses and special mode
+
+A huge thanks to all people who helped shaping this project to what it is now and don't want to be mentioned here.
+

entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [[ -z "${CFG_DEV}" ]]; then
+    echo Production mode
+    python3 index.py
+else
+    echo Development mode
+    python3 dbutils.py create
+    nodemon -w aime --legacy-watch index.py
+fi
+

example_config/cardmaker.yaml

@@ -1,3 +1,13 @@
 server:
   enable: True
   loglevel: "info"
+
+version:
+  0:
+    ongeki: 1.30.01
+    chuni: 2.00.00
+    maimai: 1.20.00
+  1:
+    ongeki: 1.35.03
+    chuni: 2.10.00
+    maimai: 1.30.00

example_config/chuni.yaml

@@ -1,9 +1,10 @@
 server:
   enable: True
   loglevel: "info"
+  news_msg: ""
 
 team:
-  name: ARTEMiS
+  name: ARTEMiS # If this is set, all players that are not on a team will use this one by default.
 
 mods:
   use_login_bonus: True
@@ -15,6 +16,12 @@ version:
   12:
     rom: 2.05.00
     data: 2.05.00
+  13:
+    rom: 2.10.00
+    data: 2.10.00
+  14:
+    rom: 2.15.00
+    data: 2.15.00
 
 crypto:
   encrypted_only: False

example_config/core.yaml

@@ -4,12 +4,22 @@ server:
   allow_unregistered_serials: True
   name: "ARTEMiS"
   is_develop: True
+  is_using_proxy: False
+  threading: False
   log_dir: "logs"
+  check_arcade_ip: False
+  strict_ip_checking: False
 
 title:
   loglevel: "info"
   hostname: "localhost"
   port: 8080
+  port_ssl: 0
+  ssl_cert: "cert/title.crt"
+  ssl_key: "cert/title.key"
+  reboot_start_time: "04:00"
+  reboot_end_time: "05:00"
+
 
 database:
   host: "localhost"
@@ -21,6 +31,7 @@ database:
   sha2_password: False
   loglevel: "warn"
   user_table_autoincrement_start: 10000
+  enable_memcached: True
   memcached_host: "localhost"
 
 frontend:
@@ -31,6 +42,7 @@ frontend:
 allnet:
   loglevel: "info"
   port: 80
+  ip_check: False
   allow_online_updates: False
   update_cfg_folder: ""
 
@@ -44,6 +56,8 @@ aimedb:
   loglevel: "info"
   port: 22345
   key: ""
+  id_secret: ""
+  id_lifetime_seconds: 86400
 
 mucha:
   enable: False

example_config/cxb.yaml

@@ -1,9 +1,3 @@
 server:
   enable: True
-  loglevel: "info"
-  hostname: "localhost"
-  ssl_enable: False
-  port: 8082
-  port_secure: 443
-  ssl_cert: "cert/title.crt"
-  ssl_key: "cert/title.key"
+  loglevel: "info"

example_config/idac.yaml

@@ -0,0 +1,22 @@
+server:
+  enable: True
+  loglevel: "info"
+  ssl: False
+  ssl_key: "cert/idac.key"
+  ssl_cert: "cert/idac.crt"
+  matching_host: "127.0.0.1"
+  port_matching: 20000
+  port_echo1: 20001
+  port_echo2: 20002
+  port_matching_p2p: 20003
+
+stamp:
+  enable: True
+  enabled_stamps: # max 3 play stamps
+    - "touhou_remilia_scarlet"
+    - "touhou_flandre_scarlet"
+    - "touhou_sakuya_izayoi"
+
+timetrial:
+  enable: True
+  enabled_timetrial: "touhou_remilia_scarlet"

example_config/mai2.yaml

@@ -1,3 +1,14 @@
 server:
   enable: True
   loglevel: "info"
+
+deliver:
+  enable: False
+  udbdl_enable: False
+  content_folder: ""
+
+uploads:
+  photos: False
+  photos_dir: ""
+  movies: False
+  movies_dir: ""

example_config/nginx_example.conf

@@ -24,8 +24,7 @@ server {
 
 # SSL titles, comment out if you don't plan on accepting SSL titles
 server {
-	listen 443 ssl default_server;
-	listen [::]:443 ssl default_server;	
+	listen 443 ssl;
 	server_name your.hostname.here;
 
 	ssl_certificate /path/to/cert/title.crt;
@@ -55,7 +54,7 @@ server {
 	ssl_session_tickets off;
 
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
-	ssl_ciphers "ALL:@SECLEVEL=1";
+	ssl_ciphers "ALL:@SECLEVEL=0";
 	ssl_prefer_server_ciphers off;
 
 	location / {
@@ -75,7 +74,7 @@ server {
 	ssl_session_tickets off;
 
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
-	ssl_ciphers "ALL:@SECLEVEL=1";
+	ssl_ciphers "ALL:@SECLEVEL=0";
 	ssl_prefer_server_ciphers off;
 
 	location / {
@@ -85,28 +84,6 @@ server {
 	}
 }
 
-# CXB, comment this out if you don't plan on serving crossbeats.
-server {
-	listen 443 ssl;	
-	server_name cxb.hostname.here;
-	
-	ssl_certificate /path/to/cert/cxb.pem;
-	ssl_certificate_key /path/to/cert/cxb.key;
-	ssl_session_timeout 1d;
-	ssl_session_cache shared:MozSSL:10m;
-	ssl_session_tickets off;
-
-	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
-	ssl_ciphers "ALL:@SECLEVEL=1";
-	ssl_prefer_server_ciphers off;
-
-	location / {
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-		proxy_pass_request_headers on;
-		proxy_pass http://localhost:8080/SDBT/104/;
-	}
-}
-
 # Frontend, set to redirect to HTTPS. Comment out if you don't intend to use the frontend
 server {
     listen 80;
@@ -143,4 +120,4 @@ server {
 		proxy_pass_request_headers on;
 		proxy_pass http://localhost:8090/;
 	}
-}
+}

example_config/ongeki.yaml

@@ -1,6 +1,7 @@
 server:
   enable: True
   loglevel: "info"
+  use_https: False
 
 gachas:
   enabled_gachas:
@@ -29,3 +30,12 @@ gachas:
     - 1156
     - 1163
     - 1164
+
+version:
+  6:
+    card_maker: 1.30.01
+  7:
+    card_maker: 1.35.03
+
+crypto:
+  encrypted_only: False

example_config/pokken.yaml

@@ -2,8 +2,11 @@ server:
   hostname: "localhost"
   enable: True
   loglevel: "info"
-  port: 9000
-  port_stun: 9001
-  port_turn: 9002
-  port_admission: 9003
-  auto_register: True
+  auto_register: True
+  enable_matching: False
+  stun_server_host: "stunserver.stunprotocol.org"
+  stun_server_port: 3478
+
+ports:
+  game: 9000
+  admission: 9001

example_config/sao.yaml

@@ -0,0 +1,13 @@
+server:
+  enable: True
+  loglevel: "info"
+  auto_register: True
+
+crypt:
+  enable: False
+  key: ""
+  iv: ""
+
+hash:
+  verify_hash: False
+  hash_base: ""

index.py

@@ -11,7 +11,7 @@ from twisted.web import server, resource
 from twisted.internet import reactor, endpoints
 from twisted.web.http import Request
 from routes import Mapper
-
+from threading import Thread
 
 class HttpDispatcher(resource.Resource):
     def __init__(self, cfg: CoreConfig, config_dir: str):
@@ -22,8 +22,8 @@ class HttpDispatcher(resource.Resource):
         self.map_post = Mapper()
         self.logger = logging.getLogger("core")
 
-        self.allnet = AllnetServlet(cfg, config_dir)
         self.title = TitleServlet(cfg, config_dir)
+        self.allnet = AllnetServlet(cfg, config_dir)
         self.mucha = MuchaServlet(cfg, config_dir)
 
         self.map_get.connect(
@@ -36,13 +36,13 @@ class HttpDispatcher(resource.Resource):
 
         self.map_post.connect(
             "allnet_downloadorder_report",
-            "/dl/report",
+            "/report-api/Report",
             controller="allnet",
             action="handle_dlorder_report",
             conditions=dict(method=["POST"]),
         )
 
-        self.map_post.connect(
+        self.map_get.connect(
             "allnet_ping",
             "/naomitest.html",
             controller="allnet",
@@ -63,6 +63,27 @@ class HttpDispatcher(resource.Resource):
             action="handle_dlorder",
             conditions=dict(method=["POST"]),
         )
+        self.map_post.connect(
+            "allnet_loaderstaterecorder",
+            "/sys/servlet/LoaderStateRecorder",
+            controller="allnet",
+            action="handle_loaderstaterecorder",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_get.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["GET"]),
+        )
         self.map_post.connect(
             "allnet_billing",
             "/request",
@@ -78,6 +99,7 @@ class HttpDispatcher(resource.Resource):
             conditions=dict(method=["POST"]),
         )
 
+        # Maintain compatability
         self.map_post.connect(
             "mucha_boardauth",
             "/mucha/boardauth.do",
@@ -92,26 +114,64 @@ class HttpDispatcher(resource.Resource):
             action="handle_updatecheck",
             conditions=dict(method=["POST"]),
         )
+        self.map_post.connect(
+            "mucha_dlstate",
+            "/mucha/downloadstate.do",
+            controller="mucha",
+            action="handle_dlstate",
+            conditions=dict(method=["POST"]),
+        )
 
-        self.map_get.connect(
-            "title_get",
-            "/{game}/{version}/{endpoint:.*?}",
-            controller="title",
-            action="render_GET",
-            conditions=dict(method=["GET"]),
-            requirements=dict(game=R"S..."),
+        self.map_post.connect(
+            "mucha_boardauth",
+            "/mucha_front/boardauth.do",
+            controller="mucha",
+            action="handle_boardauth",
+            conditions=dict(method=["POST"]),
         )
         self.map_post.connect(
-            "title_post",
-            "/{game}/{version}/{endpoint:.*?}",
-            controller="title",
-            action="render_POST",
+            "mucha_updatacheck",
+            "/mucha_front/updatacheck.do",
+            controller="mucha",
+            action="handle_updatecheck",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "mucha_dlstate",
+            "/mucha_front/downloadstate.do",
+            controller="mucha",
+            action="handle_dlstate",
             conditions=dict(method=["POST"]),
-            requirements=dict(game=R"S..."),
         )
 
+        for code, game in self.title.title_registry.items():
+            get_matchers, post_matchers = game.get_endpoint_matchers()
+            
+            for m in get_matchers:
+                self.map_get.connect(
+                    "title_get",
+                    m[1],
+                    controller="title",
+                    action="render_GET",
+                    title=code,
+                    subaction=m[0],
+                    conditions=dict(method=["GET"]),
+                    requirements=m[2],
+                )
+                
+            for m in post_matchers:
+                self.map_post.connect(
+                    "title_post",
+                    m[1],
+                    controller="title",
+                    action="render_POST",
+                    title=code,
+                    subaction=m[0],
+                    conditions=dict(method=["POST"]),
+                    requirements=m[2],
+                )
+
     def render_GET(self, request: Request) -> bytes:
-        self.logger.debug(request.uri)
         test = self.map_get.match(request.uri.decode())
         client_ip = Utils.get_ip_addr(request)
 
@@ -161,9 +221,16 @@ class HttpDispatcher(resource.Resource):
 
         if type(ret) == str:
             return ret.encode()
-        elif type(ret) == bytes:
+        
+        elif type(ret) == bytes or type(ret) == tuple: # allow for bytes or tuple (data, response code) responses
             return ret
+        
+        elif ret is None:
+            self.logger.warning(f"None returned by controller for {request.uri.decode()} endpoint")
+            return b""
+        
         else:
+            self.logger.warning(f"Unknown data type returned by controller for {request.uri.decode()} endpoint")
             return b""
 
 
@@ -222,6 +289,7 @@ if __name__ == "__main__":
 
     allnet_server_str = f"tcp:{cfg.allnet.port}:interface={cfg.server.listen_address}"
     title_server_str = f"tcp:{cfg.title.port}:interface={cfg.server.listen_address}"
+    title_https_server_str = f"ssl:{cfg.title.port_ssl}:interface={cfg.server.listen_address}:privateKey={cfg.title.ssl_key}:certKey={cfg.title.ssl_cert}"
     adb_server_str = f"tcp:{cfg.aimedb.port}:interface={cfg.server.listen_address}"
     frontend_server_str = (
         f"tcp:{cfg.frontend.port}:interface={cfg.server.listen_address}"
@@ -255,5 +323,13 @@ if __name__ == "__main__":
         endpoints.serverFromString(reactor, title_server_str).listen(
             server.Site(dispatcher)
         )
+    
+    if cfg.title.port_ssl > 0:
+        endpoints.serverFromString(reactor, title_https_server_str).listen(
+            server.Site(dispatcher)
+        )
 
-    reactor.run()  # type: ignore
+    if cfg.server.threading:
+        Thread(target=reactor.run, args=(False,)).start()
+    else:
+        reactor.run()

read.py

@@ -43,11 +43,11 @@ class BaseReader:
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Import Game Information")
     parser.add_argument(
-        "--series",
+        "--game",
         action="store",
         type=str,
         required=True,
-        help="The game series we are importing.",
+        help="The game we are importing.",
     )
     parser.add_argument(
         "--version",
@@ -109,7 +109,7 @@ if __name__ == "__main__":
     logger.setLevel(log_lv)
     coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
 
-    if args.series is None or args.version is None:
+    if args.game is None or args.version is None:
         logger.error("Game or version not specified")
         parser.print_help()
         exit(1)
@@ -134,7 +134,7 @@ if __name__ == "__main__":
     titles = Utils.get_all_titles()
 
     for dir, mod in titles.items():
-        if args.series in mod.game_codes:
+        if args.game in mod.game_codes:
             handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra)
             handler.read()
 

readme.md

@@ -3,32 +3,39 @@ A network service emulator for games running SEGA'S ALL.NET service, and similar
 
 # Supported games
 Games listed below have been tested and confirmed working. Only game versions older then the version currently active in arcades, or games versions that have not recieved a major update in over one year, are supported.
-+ Chunithm
-    + All versions up to New!! Plus
 
-+ Crossbeats Rev
++ CHUNITHM
+    + All versions up to SUN PLUS
+
++ crossbeats REV.
     + All versions + omnimix
 
 + maimai DX
-    + All versions up to Festival
+    + All versions up to FESTiVAL PLUS
 
-+ Hatsune Miku Arcade
++ Hatsune Miku: Project DIVA Arcade
     + All versions
 
 + Card Maker
-    + 1.34.xx
-    + 1.35.xx
+    + 1.30
+    + 1.35
 
-+ Ongeki
-    + All versions up to Bright Memory
++ O.N.G.E.K.I.
+    + All versions up to bright MEMORY
 
-+ Wacca
++ WACCA
     + Lily R
     + Reverse
 
-+ Pokken
++ POKKÉN TOURNAMENT
     + Final Online
 
++ Sword Art Online Arcade (partial support)
+    + Final
+
++ Initial D THE ARCADE
+    + Season 2
+
 ## Requirements
 - python 3 (tested working with 3.9 and 3.10, other versions YMMV)
 - pip
@@ -36,7 +43,7 @@ Games listed below have been tested and confirmed working. Only game versions ol
 - mysql/mariadb server
 
 ## Setup guides
-Follow the platform-specific guides for [windows](docs/INSTALL_WINDOWS.md) and [ubuntu](docs/INSTALL_UBUNTU.md) to setup and run the server.
+Follow the platform-specific guides for [windows](docs/INSTALL_WINDOWS.md), [ubuntu](docs/INSTALL_UBUNTU.md) or [docker](docs/INSTALL_DOCKER.md) to setup and run the server.
 
 ## Game specific information
 Read [Games specific info](docs/game_specific_info.md) for all supported games, importer settings, configuration option and database upgrades.

requirements.txt

@@ -1,18 +1,22 @@
-mypy
-wheel
-twisted
-pytz
-pyyaml
-sqlalchemy==1.4.46
-mysqlclient
-pyopenssl
-service_identity
-PyCryptodome
-inflection
-coloredlogs
-pylibmc; platform_system != "Windows"
-wacky
-Routes
-bcrypt
-jinja2
-protobuf
+mypy
+wheel
+twisted
+pytz
+pyyaml
+sqlalchemy==1.4.46
+mysqlclient
+pyopenssl
+service_identity
+PyCryptodome
+inflection
+coloredlogs
+pylibmc; platform_system != "Windows"
+wacky
+Routes
+bcrypt
+jinja2
+protobuf
+autobahn
+pillow
+pyjwt
+alembic==1.13.0

titles/chuni/__init__.py

@@ -6,5 +6,5 @@ from titles.chuni.read import ChuniReader
 index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
-game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW]
-current_schema_version = 3
+game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT]
+current_schema_version = 5

titles/chuni/base.py

@@ -4,13 +4,13 @@ from datetime import datetime, timedelta
 from time import strftime
 
 import pytz
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 from core.config import CoreConfig
 from titles.chuni.const import ChuniConstants
 from titles.chuni.database import ChuniData
 from titles.chuni.config import ChuniConfig
-
+SCORE_BUFFER = {}
 
 class ChuniBase:
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
@@ -44,13 +44,15 @@ class ChuniBase:
             # check if a user already has some pogress and if not add the
             # login bonus entry
             user_login_bonus = self.data.item.get_login_bonus(
-                user_id, self.version, preset["id"]
+                user_id, self.version, preset["presetId"]
             )
             if user_login_bonus is None:
-                self.data.item.put_login_bonus(user_id, self.version, preset["id"])
+                self.data.item.put_login_bonus(
+                    user_id, self.version, preset["presetId"]
+                )
                 # yeah i'm lazy
                 user_login_bonus = self.data.item.get_login_bonus(
-                    user_id, self.version, preset["id"]
+                    user_id, self.version, preset["presetId"]
                 )
 
             # skip the login bonus entirely if its already finished
@@ -66,13 +68,13 @@ class ChuniBase:
                 last_update_date = datetime.now()
 
                 all_login_boni = self.data.static.get_login_bonus(
-                    self.version, preset["id"]
+                    self.version, preset["presetId"]
                 )
 
                 # skip the current bonus preset if no boni were found
                 if all_login_boni is None or len(all_login_boni) < 1:
-                    self.logger.warn(
-                        f"No bonus entries found for bonus preset {preset['id']}"
+                    self.logger.warning(
+                        f"No bonus entries found for bonus preset {preset['presetId']}"
                     )
                     continue
 
@@ -83,14 +85,14 @@ class ChuniBase:
                 if bonus_count > max_needed_days:
                     # assume that all login preset ids under 3000 needs to be
                     # looped, like 30 and 40 are looped, 40 does not work?
-                    if preset["id"] < 3000:
+                    if preset["presetId"] < 3000:
                         bonus_count = 1
                     else:
                         is_finished = True
 
                 # grab the item for the corresponding day
                 login_item = self.data.static.get_login_bonus_by_required_days(
-                    self.version, preset["id"], bonus_count
+                    self.version, preset["presetId"], bonus_count
                 )
                 if login_item is not None:
                     # now add the present to the database so the
@@ -108,7 +110,7 @@ class ChuniBase:
                 self.data.item.put_login_bonus(
                     user_id,
                     self.version,
-                    preset["id"],
+                    preset["presetId"],
                     bonusCount=bonus_count,
                     lastUpdateDate=last_update_date,
                     isWatched=False,
@@ -147,7 +149,7 @@ class ChuniBase:
         game_events = self.data.static.get_enabled_events(self.version)
 
         if game_events is None or len(game_events) == 0:
-            self.logger.warn("No enabled events, did you run the reader?")
+            self.logger.warning("No enabled events, did you run the reader?")
             return {
                 "type": data["type"],
                 "length": 0,
@@ -156,12 +158,18 @@ class ChuniBase:
 
         event_list = []
         for evt_row in game_events:
-            tmp = {}
-            tmp["id"] = evt_row["eventId"]
-            tmp["type"] = evt_row["type"]
-            tmp["startDate"] = "2017-12-05 07:00:00.0"
-            tmp["endDate"] = "2099-12-31 00:00:00.0"
-            event_list.append(tmp)
+            event_list.append(
+                {
+                    "id": evt_row["eventId"],
+                    "type": evt_row["type"],
+                    # actually use the startDate from the import so it
+                    # properly shows all the events when new ones are imported
+                    "startDate": datetime.strftime(
+                        evt_row["startDate"], "%Y-%m-%d %H:%M:%S"
+                    ),
+                    "endDate": "2099-12-31 00:00:00",
+                }
+            )
 
         return {
             "type": data["type"],
@@ -173,21 +181,50 @@ class ChuniBase:
         return {"type": data["type"], "length": 0, "gameIdlistList": []}
 
     def handle_get_game_message_api_request(self, data: Dict) -> Dict:
-        return {"type": data["type"], "length": "0", "gameMessageList": []}
+        return {
+            "type": data["type"], 
+            "length": 1, 
+            "gameMessageList": [{ 
+                "id": 1, 
+                "type": 1,
+                "message": f"Welcome to {self.core_cfg.server.name} network!" if not self.game_cfg.server.news_msg else self.game_cfg.server.news_msg,
+                "startDate": "2017-12-05 07:00:00.0",
+                "endDate": "2099-12-31 00:00:00.0"
+            }]
+        }
 
     def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
-        return {"type": data["type"], "gameRankingList": []}
+        rankings = self.data.score.get_rankings(self.version)
+        return {"type": data["type"], "gameRankingList": rankings}
 
     def handle_get_game_sale_api_request(self, data: Dict) -> Dict:
         return {"type": data["type"], "length": 0, "gameSaleList": []}
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        reboot_start = datetime.strftime(
-            datetime.now() - timedelta(hours=4), self.date_time_format
-        )
-        reboot_end = datetime.strftime(
-            datetime.now() - timedelta(hours=3), self.date_time_format
-        )
+        # if reboot start/end time is not defined use the default behavior of being a few hours ago
+        if self.core_cfg.title.reboot_start_time == "" or self.core_cfg.title.reboot_end_time == "":
+            reboot_start = datetime.strftime(
+                datetime.utcnow() + timedelta(hours=6), self.date_time_format
+            )
+            reboot_end = datetime.strftime(
+                datetime.utcnow() + timedelta(hours=7), self.date_time_format
+            )
+        else:
+            # get current datetime in JST
+            current_jst = datetime.now(pytz.timezone('Asia/Tokyo')).date()
+
+            # parse config start/end times into datetime
+            reboot_start_time = datetime.strptime(self.core_cfg.title.reboot_start_time, "%H:%M")
+            reboot_end_time = datetime.strptime(self.core_cfg.title.reboot_end_time, "%H:%M")
+
+            # offset datetimes with current date/time
+            reboot_start_time = reboot_start_time.replace(year=current_jst.year, month=current_jst.month, day=current_jst.day, tzinfo=pytz.timezone('Asia/Tokyo'))
+            reboot_end_time = reboot_end_time.replace(year=current_jst.year, month=current_jst.month, day=current_jst.day, tzinfo=pytz.timezone('Asia/Tokyo'))
+
+            # create strings for use in gameSetting
+            reboot_start = reboot_start_time.strftime(self.date_time_format)
+            reboot_end = reboot_end_time.strftime(self.date_time_format)
+
         return {
             "gameSetting": {
                 "dataVersion": "1.00.00",
@@ -203,7 +240,6 @@ class ChuniBase:
             "isDumpUpload": "false",
             "isAou": "false",
         }
-
     def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
         user_activity_list = self.data.profile.get_profile_activity(
             data["userId"], data["kind"]
@@ -228,29 +264,36 @@ class ChuniBase:
     def handle_get_user_character_api_request(self, data: Dict) -> Dict:
         characters = self.data.item.get_characters(data["userId"])
         if characters is None:
-            return {}
-        next_idx = -1
+            return {
+                "userId": data["userId"],
+                "length": 0,
+                "nextIndex": -1,
+                "userCharacterList": [],
+            }
 
-        characterList = []
-        for x in range(int(data["nextIndex"]), len(characters)):
+        character_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(characters)):
             tmp = characters[x]._asdict()
             tmp.pop("user")
             tmp.pop("id")
-            characterList.append(tmp)
+            character_list.append(tmp)
 
-            if len(characterList) >= int(data["maxCount"]):
+            if len(character_list) >= max_ct:
                 break
 
-        if len(characterList) >= int(data["maxCount"]) and len(characters) > int(
-            data["maxCount"]
-        ) + int(data["nextIndex"]):
-            next_idx = int(data["maxCount"]) + int(data["nextIndex"]) + 1
+        if len(characters) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = -1
 
         return {
             "userId": data["userId"],
-            "length": len(characterList),
+            "length": len(character_list),
             "nextIndex": next_idx,
-            "userCharacterList": characterList,
+            "userCharacterList": character_list,
         }
 
     def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -269,6 +312,13 @@ class ChuniBase:
             "userChargeList": charge_list,
         }
 
+    def handle_get_user_recent_player_api_request(self, data: Dict) -> Dict:
+        return {
+            "userId": data["userId"],
+            "length": 0,
+            "userRecentPlayerList": [], # playUserId, playUserName, playDate, friendPoint
+        }
+
     def handle_get_user_course_api_request(self, data: Dict) -> Dict:
         user_course_list = self.data.score.get_courses(data["userId"])
         if user_course_list is None:
@@ -280,8 +330,8 @@ class ChuniBase:
             }
 
         course_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
+        next_idx = int(data.get("nextIndex", 0))
+        max_ct = int(data.get("maxCount", 300))
 
         for x in range(next_idx, len(user_course_list)):
             tmp = user_course_list[x]._asdict()
@@ -292,8 +342,8 @@ class ChuniBase:
             if len(user_course_list) >= max_ct:
                 break
 
-        if len(user_course_list) >= max_ct:
-            next_idx = next_idx + max_ct
+        if len(user_course_list) >= next_idx + max_ct:
+            next_idx += max_ct
         else:
             next_idx = -1
 
@@ -346,13 +396,90 @@ class ChuniBase:
             "userDuelList": duel_list,
         }
 
-    def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+    def handle_get_user_rival_data_api_request(self, data: Dict) -> Dict:
+        p = self.data.profile.get_rival(data["rivalId"])
+        if p is None:
+            return {}
+        userRivalData = {
+            "rivalId": p.user,
+            "rivalName": p.userName
+        }
         return {
             "userId": data["userId"],
-            "length": 0,
+            "userRivalData": userRivalData
+        }
+    
+    def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
+        rival_id = data["rivalId"]
+        next_index = int(data["nextIndex"])
+        max_count = int(data["maxCount"])
+        user_rival_music_list = []
+
+        # Fetch all the rival music entries for the user
+        all_entries = self.data.score.get_rival_music(rival_id)
+
+        # Process the entries based on max_count and nextIndex
+        for music in all_entries:
+            music_id = music["musicId"]
+            level = music["level"]
+            score = music["scoreMax"]
+            rank = music["scoreRank"]
+
+            # Create a music entry for the current music_id if it's unique
+            music_entry = next((entry for entry in user_rival_music_list if entry["musicId"] == music_id), None)
+            if music_entry is None:
+                music_entry = {
+                    "musicId": music_id,
+                    "length": 0,
+                    "userRivalMusicDetailList": []
+                }
+                user_rival_music_list.append(music_entry)
+
+            # Create a level entry for the current level if it's unique or has a higher score
+            level_entry = next((entry for entry in music_entry["userRivalMusicDetailList"] if entry["level"] == level), None)
+            if level_entry is None:
+                level_entry = {
+                    "level": level,
+                    "scoreMax": score,
+                    "scoreRank": rank
+                }
+                music_entry["userRivalMusicDetailList"].append(level_entry)
+            elif score > level_entry["scoreMax"]:
+                level_entry["scoreMax"] = score
+                level_entry["scoreRank"] = rank
+
+        # Calculate the length for each "musicId" by counting the unique levels
+        for music_entry in user_rival_music_list:
+            music_entry["length"] = len(music_entry["userRivalMusicDetailList"])
+
+        # Prepare the result dictionary with user rival music data
+        result = {
+            "userId": data["userId"],
+            "rivalId": data["rivalId"],
+            "nextIndex": str(next_index + len(user_rival_music_list[next_index: next_index + max_count]) if max_count <= len(user_rival_music_list[next_index: next_index + max_count]) else -1),
+            "userRivalMusicList": user_rival_music_list[next_index: next_index + max_count]
+        }
+        return result
+
+    
+    def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+        user_fav_item_list = []
+
+        # still needs to be implemented on WebUI
+        # 1: Music, 2: User, 3: Character
+        fav_list = self.data.item.get_all_favorites(
+            data["userId"], self.version, fav_kind=int(data["kind"])
+        )
+        if fav_list is not None:
+            for fav in fav_list:
+                user_fav_item_list.append({"id": fav["favId"]})
+
+        return {
+            "userId": data["userId"],
+            "length": len(user_fav_item_list),
             "kind": data["kind"],
             "nextIndex": -1,
-            "userFavoriteItemList": [],
+            "userFavoriteItemList": user_fav_item_list,
         }
 
     def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
@@ -375,7 +502,7 @@ class ChuniBase:
                 "userItemList": [],
             }
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")
@@ -387,13 +514,13 @@ class ChuniBase:
         xout = kind * 10000000000 + next_idx + len(items)
 
         if len(items) < int(data["maxCount"]):
-            nextIndex = 0
+            next_idx = 0
         else:
-            nextIndex = xout
+            next_idx = xout
 
         return {
             "userId": data["userId"],
-            "nextIndex": nextIndex,
+            "nextIndex": next_idx,
             "itemKind": kind,
             "length": len(items),
             "userItemList": items,
@@ -452,6 +579,7 @@ class ChuniBase:
                 "nextIndex": -1,
                 "userMusicList": [],  # 240
             }
+
         song_list = []
         next_idx = int(data["nextIndex"])
         max_ct = int(data["maxCount"])
@@ -463,22 +591,39 @@ class ChuniBase:
             tmp.pop("id")
 
             for song in song_list:
+                score_buf = SCORE_BUFFER.get(str(data["userId"])) or []
                 if song["userMusicDetailList"][0]["musicId"] == tmp["musicId"]:
                     found = True
                     song["userMusicDetailList"].append(tmp)
                     song["length"] = len(song["userMusicDetailList"])
+                    score_buf.append(tmp["musicId"])
+                    SCORE_BUFFER[str(data["userId"])] = score_buf
 
-            if not found:
+            score_buf = SCORE_BUFFER.get(str(data["userId"])) or []
+            if not found and tmp["musicId"] not in score_buf:
                 song_list.append({"length": 1, "userMusicDetailList": [tmp]})
+                score_buf.append(tmp["musicId"])
+                SCORE_BUFFER[str(data["userId"])] = score_buf
 
             if len(song_list) >= max_ct:
                 break
-
+    
+        try:        
+            while song_list[-1]["userMusicDetailList"][0]["musicId"] == music_detail[x + 1]["musicId"]:
+                music = music_detail[x + 1]._asdict()
+                music.pop("user")
+                music.pop("id")
+                song_list[-1]["userMusicDetailList"].append(music)
+                song_list[-1]["length"] += 1
+                x += 1
+        except IndexError:
+            pass
+    
         if len(song_list) >= max_ct:
-            next_idx += max_ct
+            next_idx += len(song_list)
         else:
-            next_idx = 0
-
+            next_idx = -1
+            SCORE_BUFFER[str(data["userId"])] = []
         return {
             "userId": data["userId"],
             "length": len(song_list),
@@ -573,39 +718,95 @@ class ChuniBase:
         }
 
     def handle_get_user_team_api_request(self, data: Dict) -> Dict:
-        # TODO: use the database "chuni_profile_team" with a GUI
+        # Default values
+        team_id = 65535
         team_name = self.game_cfg.team.team_name
-        if team_name == "":
+        team_rank = 0
+
+        # Get user profile
+        profile = self.data.profile.get_profile_data(data["userId"], self.version)
+        if profile and profile["teamId"]:
+            # Get team by id
+            team = self.data.profile.get_team_by_id(profile["teamId"])
+
+            if team:
+                team_id = team["id"]
+                team_name = team["teamName"]
+                team_rank = self.data.profile.get_team_rank(team["id"])
+
+        # Don't return anything if no team name has been defined for defaults and there is no team set for the player
+        if not profile["teamId"] and team_name == "":
             return {"userId": data["userId"], "teamId": 0}
 
         return {
             "userId": data["userId"],
-            "teamId": 1,
-            "teamRank": 1,
+            "teamId": team_id,
+            "teamRank": team_rank,
             "teamName": team_name,
             "userTeamPoint": {
                 "userId": data["userId"],
-                "teamId": 1,
+                "teamId": team_id,
                 "orderId": 1,
                 "teamPoint": 1,
                 "aggrDate": data["playDate"],
             },
         }
-
+    
     def handle_get_team_course_setting_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
-            "nextIndex": 0,
+            "nextIndex": -1,
             "teamCourseSettingList": [],
         }
 
+    def handle_get_team_course_setting_api_request_proto(self, data: Dict) -> Dict:
+        return {
+            "userId": data["userId"],
+            "length": 1,
+            "nextIndex": -1,
+            "teamCourseSettingList": [
+                {
+                    "orderId": 1,
+                    "courseId": 1,
+                    "classId": 1,
+                    "ruleId": 1,
+                    "courseName": "Test",
+                    "teamCourseMusicList": [
+                        {"track": 184, "type": 1, "level": 3, "selectLevel": -1},
+                        {"track": 184, "type": 1, "level": 3, "selectLevel": -1},
+                        {"track": 184, "type": 1, "level": 3, "selectLevel": -1}
+                    ],
+                    "teamCourseRankingInfoList": [],
+                    "recodeDate": "2099-12-31 11:59:99.0",
+                    "isPlayed": False
+                }
+            ],
+        }
+
     def handle_get_team_course_rule_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
-            "nextIndex": 0,
-            "teamCourseRuleList": [],
+            "nextIndex": -1,
+            "teamCourseRuleList": []
+        }
+
+    def handle_get_team_course_rule_api_request_proto(self, data: Dict) -> Dict:
+        return {
+            "userId": data["userId"],
+            "length": 1,
+            "nextIndex": -1,
+            "teamCourseRuleList": [
+                {
+                    "recoveryLife": 0,
+                    "clearLife": 100,
+                    "damageMiss": 1,
+                    "damageAttack": 1,
+                    "damageJustice": 1,
+                    "damageJusticeC": 1
+                }
+            ],
         }
 
     def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
@@ -617,18 +818,21 @@ class ChuniBase:
                 upsert["userData"][0]["userName"] = self.read_wtf8(
                     upsert["userData"][0]["userName"]
                 )
-            except:
+            except Exception:
                 pass
 
             self.data.profile.put_profile_data(
                 user_id, self.version, upsert["userData"][0]
             )
+
         if "userDataEx" in upsert:
             self.data.profile.put_profile_data_ex(
                 user_id, self.version, upsert["userDataEx"][0]
             )
+
         if "userGameOption" in upsert:
             self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
+
         if "userGameOptionEx" in upsert:
             self.data.profile.put_profile_option_ex(
                 user_id, upsert["userGameOptionEx"][0]
@@ -672,12 +876,35 @@ class ChuniBase:
 
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
-                self.data.score.put_playlog(user_id, playlog)
+                # convert the player names to utf-8
+                if playlog["playedUserName1"] is not None:
+                  playlog["playedUserName1"] = self.read_wtf8(playlog["playedUserName1"])
+                if playlog["playedUserName2"] is not None:
+                  playlog["playedUserName2"] = self.read_wtf8(playlog["playedUserName2"])
+                if playlog["playedUserName3"] is not None:
+                  playlog["playedUserName3"] = self.read_wtf8(playlog["playedUserName3"])
+                self.data.score.put_playlog(user_id, playlog, self.version)
 
         if "userTeamPoint" in upsert:
-            # TODO: team stuff
-            pass
+            team_points = upsert["userTeamPoint"]
+            try:
+                for tp in team_points:
+                    if tp["teamId"] != '65535':
+                        # Fetch the current team data
+                        current_team = self.data.profile.get_team_by_id(tp["teamId"])
 
+                        # Calculate the new teamPoint
+                        new_team_point = int(tp["teamPoint"]) + current_team["teamPoint"]
+
+                        # Prepare the data to update
+                        team_data = {
+                            "teamPoint": new_team_point
+                        }
+
+                        # Update the team data
+                        self.data.profile.update_team(tp["teamId"], team_data)
+            except:
+                pass # Probably a better way to catch if the team is not set yet (new profiles), but let's just pass
         if "userMapAreaList" in upsert:
             for map_area in upsert["userMapAreaList"]:
                 self.data.item.put_map_area(user_id, map_area)
@@ -695,6 +922,10 @@ class ChuniBase:
                 self.data.item.put_login_bonus(
                     user_id, self.version, login["presetId"], isWatched=True
                 )
+        
+        if "userRecentPlayerList" in upsert: # TODO: Seen in Air, maybe implement sometime
+            for rp in upsert["userRecentPlayerList"]:
+                pass
 
         return {"returnCode": "1"}
 
@@ -723,4 +954,4 @@ class ChuniBase:
         return {
             "userId": data["userId"],
             "userNetBattleData": {"recentNBSelectMusicList": []},
-        }
+        }

titles/chuni/config.py

@@ -19,6 +19,12 @@ class ChuniServerConfig:
                 self.__config, "chuni", "server", "loglevel", default="info"
             )
         )
+        
+    @property
+    def news_msg(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "chuni", "server", "news_msg", default=""
+        )
 
 
 class ChuniTeamConfig:
@@ -30,6 +36,11 @@ class ChuniTeamConfig:
         return CoreConfig.get_config_field(
             self.__config, "chuni", "team", "name", default=""
         )
+    @property
+    def rank_scale(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "chuni", "team", "rank_scale", default="False"
+        )
 
 
 class ChuniModsConfig:

titles/chuni/const.py

@@ -1,6 +1,7 @@
 class ChuniConstants:
     GAME_CODE = "SDBT"
     GAME_CODE_NEW = "SDHD"
+    GAME_CODE_INT = "SDGS"
 
     CONFIG_NAME = "chuni.yaml"
 
@@ -17,23 +18,26 @@ class ChuniConstants:
     VER_CHUNITHM_PARADISE = 10
     VER_CHUNITHM_NEW = 11
     VER_CHUNITHM_NEW_PLUS = 12
-
+    VER_CHUNITHM_SUN = 13
+    VER_CHUNITHM_SUN_PLUS = 14
     VERSION_NAMES = [
-        "Chunithm",
-        "Chunithm+",
-        "Chunithm Air",
-        "Chunithm Air+",
-        "Chunithm Star",
-        "Chunithm Star+",
-        "Chunithm Amazon",
-        "Chunithm Amazon+",
-        "Chunithm Crystal",
-        "Chunithm Crystal+",
-        "Chunithm Paradise",
-        "Chunithm New!!",
-        "Chunithm New!!+",
+        "CHUNITHM",
+        "CHUNITHM PLUS",
+        "CHUNITHM AIR",
+        "CHUNITHM AIR PLUS",
+        "CHUNITHM STAR",
+        "CHUNITHM STAR  PLUS",
+        "CHUNITHM AMAZON",
+        "CHUNITHM AMAZON PLUS",
+        "CHUNITHM CRYSTAL",
+        "CHUNITHM CRYSTAL PLUS",
+        "CHUNITHM PARADISE",
+        "CHUNITHM NEW!!",
+        "CHUNITHM NEW PLUS!!",
+        "CHUNITHM SUN",
+        "CHUNITHM SUN PLUS"
     ]
 
     @classmethod
     def game_ver_to_string(cls, ver: int):
-        return cls.VERSION_NAMES[ver]
+        return cls.VERSION_NAMES[ver]

titles/chuni/index.py

@@ -11,29 +11,32 @@ from Crypto.Util.Padding import pad
 from Crypto.Protocol.KDF import PBKDF2
 from Crypto.Hash import SHA1
 from os import path
-from typing import Tuple, Dict
+from typing import Tuple, Dict, List
 
 from core import CoreConfig, Utils
-from titles.chuni.config import ChuniConfig
-from titles.chuni.const import ChuniConstants
-from titles.chuni.base import ChuniBase
-from titles.chuni.plus import ChuniPlus
-from titles.chuni.air import ChuniAir
-from titles.chuni.airplus import ChuniAirPlus
-from titles.chuni.star import ChuniStar
-from titles.chuni.starplus import ChuniStarPlus
-from titles.chuni.amazon import ChuniAmazon
-from titles.chuni.amazonplus import ChuniAmazonPlus
-from titles.chuni.crystal import ChuniCrystal
-from titles.chuni.crystalplus import ChuniCrystalPlus
-from titles.chuni.paradise import ChuniParadise
-from titles.chuni.new import ChuniNew
-from titles.chuni.newplus import ChuniNewPlus
+from core.title import BaseServlet
+from .config import ChuniConfig
+from .const import ChuniConstants
+from .base import ChuniBase
+from .plus import ChuniPlus
+from .air import ChuniAir
+from .airplus import ChuniAirPlus
+from .star import ChuniStar
+from .starplus import ChuniStarPlus
+from .amazon import ChuniAmazon
+from .amazonplus import ChuniAmazonPlus
+from .crystal import ChuniCrystal
+from .crystalplus import ChuniCrystalPlus
+from .paradise import ChuniParadise
+from .new import ChuniNew
+from .newplus import ChuniNewPlus
+from .sun import ChuniSun
+from .sunplus import ChuniSunPlus
 
 
-class ChuniServlet:
+class ChuniServlet(BaseServlet):
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
-        self.core_cfg = core_cfg
+        super().__init__(core_cfg, cfg_dir)
         self.game_cfg = ChuniConfig()
         self.hash_table: Dict[Dict[str, str]] = {}
         if path.exists(f"{cfg_dir}/{ChuniConstants.CONFIG_NAME}"):
@@ -55,6 +58,8 @@ class ChuniServlet:
             ChuniParadise,
             ChuniNew,
             ChuniNewPlus,
+            ChuniSun,
+            ChuniSunPlus,
         ]
 
         self.logger = logging.getLogger("chuni")
@@ -96,24 +101,42 @@ class ChuniServlet:
             ]
             for method in method_list:
                 method_fixed = inflection.camelize(method)[6:-7]
+                # number of iterations was changed to 70 in SUN and then to 36
+                if version == ChuniConstants.VER_CHUNITHM_SUN_PLUS:
+                    iter_count = 36
+                elif version == ChuniConstants.VER_CHUNITHM_SUN:
+                    iter_count = 70
+                else:
+                    iter_count = 44
+                    
                 hash = PBKDF2(
                     method_fixed,
                     bytes.fromhex(keys[2]),
                     128,
-                    count=44,
+                    count=iter_count,
                     hmac_hash_module=SHA1,
                 )
 
-                self.hash_table[version][hash.hex()] = method_fixed
+                hashed_name = hash.hex()[:32] # truncate unused bytes like the game does
+                self.hash_table[version][hashed_name] = method_fixed
 
                 self.logger.debug(
                     f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()}"
                 )
 
+    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
+        return (
+            [], 
+            [
+               ("render_POST", "/{game}/{version}/ChuniServlet/{endpoint}", {}),
+               ("render_POST", "/{game}/{version}/ChuniServlet/MatchingServer/{endpoint}", {})
+            ]
+        )
+
     @classmethod
-    def get_allnet_info(
+    def is_game_enabled(
         cls, game_code: str, core_cfg: CoreConfig, cfg_dir: str
-    ) -> Tuple[bool, str, str]:
+    ) -> bool:
         game_cfg = ChuniConfig()
         if path.exists(f"{cfg_dir}/{ChuniConstants.CONFIG_NAME}"):
             game_cfg.update(
@@ -121,54 +144,72 @@ class ChuniServlet:
             )
 
         if not game_cfg.server.enable:
-            return (False, "", "")
+            return False
 
-        if core_cfg.server.is_develop:
-            return (
-                True,
-                f"http://{core_cfg.title.hostname}:{core_cfg.title.port}/{game_code}/$v/",
-                "",
-            )
+        return True
 
-        return (True, f"http://{core_cfg.title.hostname}/{game_code}/$v/", "")
+    def get_allnet_info(self, game_code: str, game_ver: int, keychip: str) -> Tuple[str, str]:
+        if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
+            return (f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", self.core_cfg.title.hostname)
 
-    def render_POST(self, request: Request, version: int, url_path: str) -> bytes:
-        if url_path.lower() == "ping":
+        return (f"http://{self.core_cfg.title.hostname}/{game_code}/{game_ver}/", self.core_cfg.title.hostname)
+
+    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+        endpoint = matchers['endpoint']
+        version = int(matchers['version'])
+        game_code = matchers['game']
+
+        if endpoint.lower() == "ping":
             return zlib.compress(b'{"returnCode": "1"}')
 
         req_raw = request.content.getvalue()
-        url_split = url_path.split("/")
+
         encrtped = False
         internal_ver = 0
-        endpoint = url_split[len(url_split) - 1]
         client_ip = Utils.get_ip_addr(request)
 
-        if version < 105:  # 1.0
-            internal_ver = ChuniConstants.VER_CHUNITHM
-        elif version >= 105 and version < 110:  # Plus
-            internal_ver = ChuniConstants.VER_CHUNITHM_PLUS
-        elif version >= 110 and version < 115:  # Air
-            internal_ver = ChuniConstants.VER_CHUNITHM_AIR
-        elif version >= 115 and version < 120:  # Air Plus
-            internal_ver = ChuniConstants.VER_CHUNITHM_AIR_PLUS
-        elif version >= 120 and version < 125:  # Star
-            internal_ver = ChuniConstants.VER_CHUNITHM_STAR
-        elif version >= 125 and version < 130:  # Star Plus
-            internal_ver = ChuniConstants.VER_CHUNITHM_STAR_PLUS
-        elif version >= 130 and version < 135:  # Amazon
-            internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON
-        elif version >= 135 and version < 140:  # Amazon Plus
-            internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
-        elif version >= 140 and version < 145:  # Crystal
-            internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL
-        elif version >= 145 and version < 150:  # Crystal Plus
-            internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
-        elif version >= 150 and version < 200:  # Paradise
-            internal_ver = ChuniConstants.VER_CHUNITHM_PARADISE
-        elif version >= 200 and version < 205:  # New
-            internal_ver = ChuniConstants.VER_CHUNITHM_NEW
-        elif version >= 205 and version < 210:  # New Plus
-            internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS
+        if game_code == "SDHD" or game_code == "SDBT": # JP
+          if version < 105:  # 1.0
+              internal_ver = ChuniConstants.VER_CHUNITHM
+          elif version >= 105 and version < 110:  # PLUS
+              internal_ver = ChuniConstants.VER_CHUNITHM_PLUS
+          elif version >= 110 and version < 115:  # AIR
+              internal_ver = ChuniConstants.VER_CHUNITHM_AIR
+          elif version >= 115 and version < 120:  # AIR PLUS
+              internal_ver = ChuniConstants.VER_CHUNITHM_AIR_PLUS
+          elif version >= 120 and version < 125:  # STAR
+              internal_ver = ChuniConstants.VER_CHUNITHM_STAR
+          elif version >= 125 and version < 130:  # STAR PLUS
+              internal_ver = ChuniConstants.VER_CHUNITHM_STAR_PLUS
+          elif version >= 130 and version < 135:  # AMAZON
+              internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON
+          elif version >= 135 and version < 140:  # AMAZON PLUS
+              internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
+          elif version >= 140 and version < 145:  # CRYSTAL
+              internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL
+          elif version >= 145 and version < 150:  # CRYSTAL PLUS
+              internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
+          elif version >= 150 and version < 200:  # PARADISE
+              internal_ver = ChuniConstants.VER_CHUNITHM_PARADISE
+          elif version >= 200 and version < 205:  # NEW!!
+              internal_ver = ChuniConstants.VER_CHUNITHM_NEW
+          elif version >= 205 and version < 210:  # NEW PLUS!!
+              internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS
+          elif version >= 210 and version < 215:  # SUN
+              internal_ver = ChuniConstants.VER_CHUNITHM_SUN
+          elif version >= 215: # SUN
+              internal_ver = ChuniConstants.VER_CHUNITHM_SUN_PLUS
+        elif game_code == "SDGS": # Int
+          if version < 110: # SUPERSTAR
+              internal_ver = ChuniConstants.PARADISE
+          elif version >= 110 and version < 115: # NEW
+              internal_ver = ChuniConstants.VER_CHUNITHM_NEW
+          elif version >= 115 and version < 120: # NEW PLUS!!
+              internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS
+          elif version >= 120 and version < 125: # SUN
+              internal_ver = ChuniConstants.VER_CHUNITHM_SUN
+          elif version >= 125: # SUN PLUS
+              internal_ver = ChuniConstants.VER_CHUNITHM_SUN_PLUS
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
             # If we get a 32 character long hex string, it's a hash and we're
@@ -233,6 +274,7 @@ class ChuniServlet:
         self.logger.info(f"v{version} {endpoint} request from {client_ip}")
         self.logger.debug(req_data)
 
+        endpoint = endpoint.replace("C3Exp", "") if game_code == "SDGS" else endpoint
         func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"
         handler_cls = self.versions[internal_ver](self.core_cfg, self.game_cfg)
 
@@ -267,4 +309,4 @@ class ChuniServlet:
             bytes.fromhex(self.game_cfg.crypto.keys[internal_ver][1]),
         )
 
-        return crypt.encrypt(padded)
+        return crypt.encrypt(padded)

titles/chuni/new.py

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
 from random import randint
 from typing import Dict
 
+import pytz
 from core.config import CoreConfig
 from titles.chuni.const import ChuniConstants
 from titles.chuni.database import ChuniData
@@ -23,41 +24,61 @@ class ChuniNew(ChuniBase):
         self.version = ChuniConstants.VER_CHUNITHM_NEW
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        # use UTC time and convert it to JST time by adding +9
+        # matching therefore starts one hour before and lasts for 8 hours
         match_start = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=8), self.date_time_format
         )
         match_end = datetime.strftime(
-            datetime.now() + timedelta(hours=10), self.date_time_format
-        )
-        reboot_start = datetime.strftime(
-            datetime.now() - timedelta(hours=11), self.date_time_format
-        )
-        reboot_end = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=16), self.date_time_format
         )
+        # if reboot start/end time is not defined use the default behavior of being a few hours ago
+        if self.core_cfg.title.reboot_start_time == "" or self.core_cfg.title.reboot_end_time == "":
+            reboot_start = datetime.strftime(
+                datetime.utcnow() + timedelta(hours=6), self.date_time_format
+            )
+            reboot_end = datetime.strftime(
+                datetime.utcnow() + timedelta(hours=7), self.date_time_format
+            )
+        else:
+            # get current datetime in JST
+            current_jst = datetime.now(pytz.timezone('Asia/Tokyo')).date()
+
+            # parse config start/end times into datetime
+            reboot_start_time = datetime.strptime(self.core_cfg.title.reboot_start_time, "%H:%M")
+            reboot_end_time = datetime.strptime(self.core_cfg.title.reboot_end_time, "%H:%M")
+
+            # offset datetimes with current date/time
+            reboot_start_time = reboot_start_time.replace(year=current_jst.year, month=current_jst.month, day=current_jst.day, tzinfo=pytz.timezone('Asia/Tokyo'))
+            reboot_end_time = reboot_end_time.replace(year=current_jst.year, month=current_jst.month, day=current_jst.day, tzinfo=pytz.timezone('Asia/Tokyo'))
+
+            # create strings for use in gameSetting
+            reboot_start = reboot_start_time.strftime(self.date_time_format)
+            reboot_end = reboot_end_time.strftime(self.date_time_format)
         return {
             "gameSetting": {
-                "isMaintenance": "false",
+                "isMaintenance": False,
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,
-                "isBackgroundDistribute": "false",
+                "isBackgroundDistribute": False,
                 "maxCountCharacter": 300,
                 "maxCountItem": 300,
                 "maxCountMusic": 300,
                 "matchStartTime": match_start,
                 "matchEndTime": match_end,
-                "matchTimeLimit": 99,
+                "matchTimeLimit": 60,
                 "matchErrorLimit": 9999,
                 "romVersion": self.game_cfg.version.version(self.version)["rom"],
                 "dataVersion": self.game_cfg.version.version(self.version)["data"],
                 "matchingUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "matchingUriX": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                # might be really important for online battle to connect the cabs via UDP port 50201
                 "udpHolePunchUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "reflectorUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
             },
-            "isDumpUpload": "false",
-            "isAou": "false",
+            "isDumpUpload": False,
+            "isAou": False,
         }
 
     def handle_remove_token_api_request(self, data: Dict) -> Dict:
@@ -468,3 +489,162 @@ class ChuniNew(ChuniBase):
             self.data.item.put_user_print_state(user_id, id=order_id, hasCompleted=True)
 
         return {"returnCode": "1", "apiName": "CMUpsertUserPrintCancelApi"}
+
+    def handle_ping_request(self, data: Dict) -> Dict:
+        # matchmaking ping request
+        return {"returnCode": "1"}
+
+    def handle_begin_matching_api_request(self, data: Dict) -> Dict:
+        room_id = 1
+        # check if there is a free matching room
+        matching_room = self.data.item.get_oldest_free_matching(self.version)
+
+        if matching_room is None:
+            # grab the latest roomId and add 1 for the new room
+            newest_matching = self.data.item.get_newest_matching(self.version)
+            if newest_matching is not None:
+                room_id = newest_matching["roomId"] + 1
+
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+
+            # create the new room with room_id and the current user id (host)
+            # user id is required for the countdown later on
+            self.data.item.put_matching(
+                self.version, room_id, [new_member], user_id=new_member["userId"]
+            )
+
+            # get the newly created matching room
+            matching_room = self.data.item.get_matching(self.version, room_id)
+        else:
+            # a room already exists, so just add the new member to it
+            matching_member_list = matching_room["matchingMemberInfoList"]
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+            matching_member_list.append(new_member)
+
+            # add the updated room to the database, make sure to set isFull correctly!
+            self.data.item.put_matching(
+                self.version,
+                matching_room["roomId"],
+                matching_member_list,
+                user_id=matching_room["user"],
+                is_full=True if len(matching_member_list) >= 4 else False,
+            )
+
+        matching_wait = {
+            "isFinish": False,
+            "restMSec": matching_room["restMSec"],  # in sec
+            "pollingInterval": 1,  # in sec
+            "matchingMemberInfoList": matching_room["matchingMemberInfoList"],
+        }
+
+        return {"roomId": 1, "matchingWaitState": matching_wait}
+
+    def handle_end_matching_api_request(self, data: Dict) -> Dict:
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+
+        # only set the host user to role 1 every other to 0?
+        role_list = [
+            {"role": 1} if m["userId"] == matching_room["user"] else {"role": 0}
+            for m in members
+        ]
+
+        self.data.item.put_matching(
+            self.version,
+            matching_room["roomId"],
+            members,
+            user_id=matching_room["user"],
+            rest_sec=0,  # make sure to always set 0
+            is_full=True,  # and full, so no one can join
+        )
+
+        return {
+            "matchingResult": 1,  # needs to be 1 for successful matching
+            "matchingMemberInfoList": members,
+            # no idea, maybe to differentiate between CPUs and real players?
+            "matchingMemberRoleList": role_list,
+            # TCP/UDP connection?
+            "reflectorUri": f"{self.core_cfg.title.hostname}",
+        }
+
+    def handle_remove_matching_member_api_request(self, data: Dict) -> Dict:
+        # get all matching rooms, because Chuni only returns the userId
+        # not the actual roomId
+        matching_rooms = self.data.item.get_all_matchings(self.version)
+        if matching_rooms is None:
+            return {"returnCode": "1"}
+
+        for room in matching_rooms:
+            old_members = room["matchingMemberInfoList"]
+            new_members = [m for m in old_members if m["userId"] != data["userId"]]
+
+            # if nothing changed go to the next room
+            if len(old_members) == len(new_members):
+                continue
+
+            # if the last user got removed, delete the matching room
+            if len(new_members) <= 0:
+                self.data.item.delete_matching(self.version, room["roomId"])
+            else:
+                # remove the user from the room
+                self.data.item.put_matching(
+                    self.version,
+                    room["roomId"],
+                    new_members,
+                    user_id=room["user"],
+                    rest_sec=room["restMSec"],
+                )
+
+        return {"returnCode": "1"}
+
+    def handle_get_matching_state_api_request(self, data: Dict) -> Dict:
+        polling_interval = 1
+        # get the current active room
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+        rest_sec = matching_room["restMSec"]
+
+        # grab the current member
+        current_member = data["matchingMemberInfo"]
+
+        # only the host user can decrease the countdown
+        if matching_room["user"] == int(current_member["userId"]):
+            # cap the restMSec to 0
+            if rest_sec > 0:
+                rest_sec -= polling_interval
+            else:
+                rest_sec = 0
+
+        # update the members in order to recieve messages
+        for i, member in enumerate(members):
+            if member["userId"] == current_member["userId"]:
+                # replace the old user data with the current user data,
+                # also parse WTF-8 everytime
+                current_member["userName"] = self.read_wtf8(current_member["userName"])
+                members[i] = current_member
+
+        self.data.item.put_matching(
+            self.version,
+            data["roomId"],
+            members,
+            rest_sec=rest_sec,
+            user_id=matching_room["user"],
+        )
+
+        # only add the other members to the list
+        diff_members = [m for m in members if m["userId"] != current_member["userId"]]
+
+        matching_wait = {
+            # makes no difference? Always use False?
+            "isFinish": True if rest_sec == 0 else False,
+            "restMSec": rest_sec,
+            "pollingInterval": polling_interval,
+            # the current user needs to be the first one?
+            "matchingMemberInfoList": [current_member] + diff_members,
+        }
+
+        return {"matchingWaitState": matching_wait}

titles/chuni/newplus.py

@@ -36,6 +36,6 @@ class ChuniNewPlus(ChuniNew):
     def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
         user_data = super().handle_cm_get_user_preview_api_request(data)
 
-        # hardcode lastDataVersion for CardMaker 1.35
+        # hardcode lastDataVersion for CardMaker 1.35 A028
         user_data["lastDataVersion"] = "2.05.00"
-        return user_data
+        return user_data

titles/chuni/read.py

@@ -67,7 +67,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted login bonus preset {id}")
                     else:
-                        self.logger.warn(f"Failed to insert login bonus preset {id}")
+                        self.logger.warning(f"Failed to insert login bonus preset {id}")
 
                     for bonus in xml_root.find("infos").findall("LoginBonusDataInfo"):
                         for name in bonus.findall("loginBonusName"):
@@ -113,7 +113,7 @@ class ChuniReader(BaseReader):
                                 if result is not None:
                                     self.logger.info(f"Inserted login bonus {bonus_id}")
                                 else:
-                                    self.logger.warn(
+                                    self.logger.warning(
                                         f"Failed to insert login bonus {bonus_id}"
                                     )
 
@@ -138,7 +138,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted event {id}")
                     else:
-                        self.logger.warn(f"Failed to insert event {id}")
+                        self.logger.warning(f"Failed to insert event {id}")
 
     def read_music(self, music_dir: str) -> None:
         for root, dirs, files in walk(music_dir):
@@ -169,8 +169,10 @@ class ChuniReader(BaseReader):
                             fumen_path = MusicFumenData.find("file").find("path")
 
                             if fumen_path is not None:
-                                chart_id = MusicFumenData.find("type").find("id").text
-                                if chart_id == "4":
+                                chart_type = MusicFumenData.find("type")
+                                chart_id = chart_type.find("id").text
+                                chart_diff = chart_type.find("str").text
+                                if chart_diff == "WorldsEnd" and (chart_id == "4" or chart_id == "5"): # 4 in SDBT, 5 in SDHD
                                     level = float(xml_root.find("starDifType").text)
                                     we_chara = (
                                         xml_root.find("worldsEndTagName")
@@ -200,7 +202,7 @@ class ChuniReader(BaseReader):
                                         f"Inserted music {song_id} chart {chart_id}"
                                     )
                                 else:
-                                    self.logger.warn(
+                                    self.logger.warning(
                                         f"Failed to insert music {song_id} chart {chart_id}"
                                     )
 
@@ -232,7 +234,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted charge {id}")
                     else:
-                        self.logger.warn(f"Failed to insert charge {id}")
+                        self.logger.warning(f"Failed to insert charge {id}")
 
     def read_avatar(self, avatar_dir: str) -> None:
         for root, dirs, files in walk(avatar_dir):
@@ -259,4 +261,4 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted avatarAccessory {id}")
                     else:
-                        self.logger.warn(f"Failed to insert avatarAccessory {id}")
+                        self.logger.warning(f"Failed to insert avatarAccessory {id}")

titles/chuni/schema/item.py

@@ -1,10 +1,16 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+    delete,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.schema import ForeignKey
-from sqlalchemy.sql import func, select
-from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.sql import func, select, insert
 from sqlalchemy.engine import Row
 
 from core.data.schema import BaseData, metadata
@@ -203,18 +209,144 @@ login_bonus = Table(
     mysql_charset="utf8mb4",
 )
 
+favorite = Table(
+    "chuni_item_favorite",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("favId", Integer, nullable=False),
+    Column("favKind", Integer, nullable=False, server_default="1"),
+    UniqueConstraint("version", "user", "favId", name="chuni_item_favorite_uk"),
+    mysql_charset="utf8mb4",
+)
+
+matching = Table(
+    "chuni_item_matching",
+    metadata,
+    Column("roomId", Integer, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("restMSec", Integer, nullable=False, server_default="60"),
+    Column("isFull", Boolean, nullable=False, server_default="0"),
+    PrimaryKeyConstraint("roomId", "version", name="chuni_item_matching_pk"),
+    Column("matchingMemberInfoList", JSON, nullable=False),
+    mysql_charset="utf8mb4",
+)
+
 
 class ChuniItemData(BaseData):
+    def get_oldest_free_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version,
+                matching.c.isFull == False
+            )
+        ).order_by(matching.c.roomId.asc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_newest_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        ).order_by(matching.c.roomId.desc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_all_matchings(self, version: int) -> Optional[List[Row]]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_matching(self, version: int, room_id: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(matching.c.version == version, matching.c.roomId == room_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_matching(
+        self,
+        version: int,
+        room_id: int,
+        matching_member_info_list: List,
+        user_id: int = None,
+        rest_sec: int = 60,
+        is_full: bool = False
+    ) -> Optional[int]:
+        result = self.upsert(matching, dict(
+            roomId=room_id,
+            version=version,
+            restMSec=rest_sec,
+            user=user_id,
+            isFull=is_full,
+            matchingMemberInfoList=matching_member_info_list,
+        ))
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def delete_matching(self, version: int, room_id: int):
+        sql = delete(matching).where(
+            and_(matching.c.roomId == room_id, matching.c.version == version)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def get_all_favorites(
+        self, user_id: int, version: int, fav_kind: int = 1
+    ) -> Optional[List[Row]]:
+        sql = favorite.select(
+            and_(
+                favorite.c.version == version,
+                favorite.c.user == user_id,
+                favorite.c.favKind == fav_kind,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def put_login_bonus(
         self, user_id: int, version: int, preset_id: int, **login_bonus_data
     ) -> Optional[int]:
-        sql = insert(login_bonus).values(
-            version=version, user=user_id, presetId=preset_id, **login_bonus_data
-        )
-
-        conflict = sql.on_duplicate_key_update(presetId=preset_id, **login_bonus_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(login_bonus, dict(
+            version=version,
+            user=user_id,
+            presetId=preset_id,
+            **login_bonus_data
+        ))
         if result is None:
             return None
         return result.lastrowid
@@ -256,10 +388,7 @@ class ChuniItemData(BaseData):
 
         character_data = self.fix_bools(character_data)
 
-        sql = insert(character).values(**character_data)
-        conflict = sql.on_duplicate_key_update(**character_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(character, character_data)
         if result is None:
             return None
         return result.lastrowid
@@ -287,10 +416,7 @@ class ChuniItemData(BaseData):
 
         item_data = self.fix_bools(item_data)
 
-        sql = insert(item).values(**item_data)
-        conflict = sql.on_duplicate_key_update(**item_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(item, item_data)
         if result is None:
             return None
         return result.lastrowid
@@ -313,10 +439,8 @@ class ChuniItemData(BaseData):
 
         duel_data = self.fix_bools(duel_data)
 
-        sql = insert(duel).values(**duel_data)
-        conflict = sql.on_duplicate_key_update(**duel_data)
+        result = self.upsert(duel, duel_data)
 
-        result = self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
@@ -334,10 +458,7 @@ class ChuniItemData(BaseData):
 
         map_data = self.fix_bools(map_data)
 
-        sql = insert(map).values(**map_data)
-        conflict = sql.on_duplicate_key_update(**map_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(map, map_data)
         if result is None:
             return None
         return result.lastrowid
@@ -355,10 +476,7 @@ class ChuniItemData(BaseData):
 
         map_area_data = self.fix_bools(map_area_data)
 
-        sql = insert(map_area).values(**map_area_data)
-        conflict = sql.on_duplicate_key_update(**map_area_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(map_area, map_area_data)
         if result is None:
             return None
         return result.lastrowid
@@ -382,15 +500,9 @@ class ChuniItemData(BaseData):
     def put_user_gacha(
         self, aime_id: int, gacha_id: int, gacha_data: Dict
     ) -> Optional[int]:
-        sql = insert(gacha).values(user=aime_id, gachaId=gacha_id, **gacha_data)
-
-        conflict = sql.on_duplicate_key_update(
-            user=aime_id, gachaId=gacha_id, **gacha_data
-        )
-        result = self.execute(conflict)
-
+        result = self.upsert(gacha, dict(user=aime_id, gachaId=gacha_id, **gacha_data))
         if result is None:
-            self.logger.warn(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
+            self.logger.warning(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -426,13 +538,9 @@ class ChuniItemData(BaseData):
         return result.fetchall()
 
     def put_user_print_state(self, aime_id: int, **print_data) -> Optional[int]:
-        sql = insert(print_state).values(user=aime_id, **print_data)
-
-        conflict = sql.on_duplicate_key_update(user=aime_id, **print_data)
-        result = self.execute(conflict)
-
+        result = self.upsert(print_state, dict(user=aime_id, **print_data))
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_state: Failed to insert! aime_id: {aime_id}"
             )
             return None
@@ -441,15 +549,9 @@ class ChuniItemData(BaseData):
     def put_user_print_detail(
         self, aime_id: int, serial_id: str, user_print_data: Dict
     ) -> Optional[int]:
-        sql = insert(print_detail).values(
-            user=aime_id, serialId=serial_id, **user_print_data
-        )
-
-        conflict = sql.on_duplicate_key_update(user=aime_id, **user_print_data)
-        result = self.execute(conflict)
-
+        result = self.upsert(print_detail, dict(user=aime_id, serialId=serial_id, **user_print_data))
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_detail: Failed to insert! aime_id: {aime_id}"
             )
             return None

titles/chuni/schema/profile.py

@@ -4,8 +4,7 @@ from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteg
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.engine import Row
-from sqlalchemy.sql import func, select
-from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.sql import func, select, insert
 
 from core.data.schema import BaseData, metadata
 
@@ -89,8 +88,6 @@ profile = Table(
         Integer,
         ForeignKey("chuni_profile_team.id", ondelete="SET NULL", onupdate="SET NULL"),
     ),
-    Column("avatarBack", Integer, server_default="0"),
-    Column("avatarFace", Integer, server_default="0"),
     Column("eliteRankPoint", Integer, server_default="0"),
     Column("stockedGridCount", Integer, server_default="0"),
     Column("netBattleLoseCount", Integer, server_default="0"),
@@ -98,10 +95,8 @@ profile = Table(
     Column("netBattle4thCount", Integer, server_default="0"),
     Column("overPowerRate", Integer, server_default="0"),
     Column("battleRewardStatus", Integer, server_default="0"),
-    Column("avatarPoint", Integer, server_default="0"),
     Column("netBattle1stCount", Integer, server_default="0"),
     Column("charaIllustId", Integer, server_default="0"),
-    Column("avatarItem", Integer, server_default="0"),
     Column("userNameEx", String(8), server_default=""),
     Column("netBattleWinCount", Integer, server_default="0"),
     Column("netBattleCorrection", Integer, server_default="0"),
@@ -112,7 +107,6 @@ profile = Table(
     Column("netBattle3rdCount", Integer, server_default="0"),
     Column("netBattleConsecutiveWinCount", Integer, server_default="0"),
     Column("overPowerLowerRank", Integer, server_default="0"),
-    Column("avatarWear", Integer, server_default="0"),
     Column("classEmblemBase", Integer, server_default="0"),
     Column("battleRankPoint", Integer, server_default="0"),
     Column("netBattle2ndCount", Integer, server_default="0"),
@@ -120,13 +114,19 @@ profile = Table(
     Column("skillId", Integer, server_default="0"),
     Column("lastCountryCode", String(5), server_default="JPN"),
     Column("isNetBattleHost", Boolean, server_default="0"),
-    Column("avatarFront", Integer, server_default="0"),
-    Column("avatarSkin", Integer, server_default="0"),
     Column("battleRewardCount", Integer, server_default="0"),
     Column("battleRewardIndex", Integer, server_default="0"),
     Column("netBattlePlayCount", Integer, server_default="0"),
     Column("exMapLoopCount", Integer, server_default="0"),
     Column("netBattleEndState", Integer, server_default="0"),
+    Column("rankUpChallengeResults", JSON),
+    Column("avatarBack", Integer, server_default="0"),
+    Column("avatarFace", Integer, server_default="0"),
+    Column("avatarPoint", Integer, server_default="0"),
+    Column("avatarItem", Integer, server_default="0"),
+    Column("avatarWear", Integer, server_default="0"),
+    Column("avatarFront", Integer, server_default="0"),
+    Column("avatarSkin", Integer, server_default="0"),
     Column("avatarHead", Integer, server_default="0"),
     UniqueConstraint("user", "version", name="chuni_profile_profile_uk"),
     mysql_charset="utf8mb4",
@@ -404,12 +404,9 @@ class ChuniProfileData(BaseData):
 
         profile_data = self.fix_bools(profile_data)
 
-        sql = insert(profile).values(**profile_data)
-        conflict = sql.on_duplicate_key_update(**profile_data)
-        result = self.execute(conflict)
-
+        result = self.upsert(profile, profile_data)
         if result is None:
-            self.logger.warn(f"put_profile_data: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_profile_data: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -417,8 +414,8 @@ class ChuniProfileData(BaseData):
         sql = (
             select([profile, option])
             .join(option, profile.c.user == option.c.user)
-            .filter(and_(profile.c.user == aime_id, profile.c.version == version))
-        )
+            .filter(and_(profile.c.user == aime_id, profile.c.version <= version))
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -429,9 +426,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile).where(
             and_(
                 profile.c.user == aime_id,
-                profile.c.version == version,
+                profile.c.version <= version,
             )
-        )
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -446,12 +443,10 @@ class ChuniProfileData(BaseData):
         if "accessCode" in profile_ex_data:
             profile_ex_data.pop("accessCode")
 
-        sql = insert(profile_ex).values(**profile_ex_data)
-        conflict = sql.on_duplicate_key_update(**profile_ex_data)
-        result = self.execute(conflict)
+        result = self.upsert(profile_ex, profile_ex_data)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_data_ex: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -461,9 +456,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile_ex).where(
             and_(
                 profile_ex.c.user == aime_id,
-                profile_ex.c.version == version,
+                profile_ex.c.version <= version,
             )
-        )
+        ).order_by(profile_ex.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -473,12 +468,10 @@ class ChuniProfileData(BaseData):
     def put_profile_option(self, aime_id: int, option_data: Dict) -> Optional[int]:
         option_data["user"] = aime_id
 
-        sql = insert(option).values(**option_data)
-        conflict = sql.on_duplicate_key_update(**option_data)
-        result = self.execute(conflict)
+        result = self.upsert(option, option_data)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_option: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -497,12 +490,10 @@ class ChuniProfileData(BaseData):
     ) -> Optional[int]:
         option_ex_data["user"] = aime_id
 
-        sql = insert(option_ex).values(**option_ex_data)
-        conflict = sql.on_duplicate_key_update(**option_ex_data)
-        result = self.execute(conflict)
+        result = self.upsert(option_ex, option_ex_data)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_option_ex: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -519,14 +510,10 @@ class ChuniProfileData(BaseData):
     def put_profile_recent_rating(
         self, aime_id: int, recent_rating_data: List[Dict]
     ) -> Optional[int]:
-        sql = insert(recent_rating).values(
-            user=aime_id, recentRating=recent_rating_data
-        )
-        conflict = sql.on_duplicate_key_update(recentRating=recent_rating_data)
+        result = self.upsert(recent_rating, dict(user=aime_id, recentRating=recent_rating_data))
 
-        result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_recent_rating: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -546,12 +533,10 @@ class ChuniProfileData(BaseData):
         activity_data["activityId"] = activity_data["id"]
         activity_data.pop("id")
 
-        sql = insert(activity).values(**activity_data)
-        conflict = sql.on_duplicate_key_update(**activity_data)
-        result = self.execute(conflict)
+        result = self.upsert(activity, activity_data)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_activity: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -572,12 +557,10 @@ class ChuniProfileData(BaseData):
     def put_profile_charge(self, aime_id: int, charge_data: Dict) -> Optional[int]:
         charge_data["user"] = aime_id
 
-        sql = insert(charge).values(**charge_data)
-        conflict = sql.on_duplicate_key_update(**charge_data)
-        result = self.execute(conflict)
+        result = self.upsert(charge, charge_data)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_charge: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -600,10 +583,7 @@ class ChuniProfileData(BaseData):
     def put_profile_emoney(self, aime_id: int, emoney_data: Dict) -> Optional[int]:
         emoney_data["user"] = aime_id
 
-        sql = insert(emoney).values(**emoney_data)
-        conflict = sql.on_duplicate_key_update(**emoney_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(emoney, emoney_data)
         if result is None:
             return None
         return result.lastrowid
@@ -621,10 +601,7 @@ class ChuniProfileData(BaseData):
     ) -> Optional[int]:
         overpower_data["user"] = aime_id
 
-        sql = insert(overpower).values(**overpower_data)
-        conflict = sql.on_duplicate_key_update(**overpower_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(overpower, overpower_data)
         if result is None:
             return None
         return result.lastrowid
@@ -636,3 +613,66 @@ class ChuniProfileData(BaseData):
         if result is None:
             return None
         return result.fetchall()
+
+    def get_team_by_id(self, team_id: int) -> Optional[Row]:
+        sql = select(team).where(team.c.id == team_id)
+        result = self.execute(sql)
+
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_team_rank(self, team_id: int) -> int:
+        # Normal ranking system, likely the one used in the real servers
+        # Query all teams sorted by 'teamPoint'
+        result = self.execute(
+            select(team.c.id).order_by(team.c.teamPoint.desc())
+        )
+
+        # Get the rank of the team with the given team_id
+        rank = None
+        for i, row in enumerate(result, start=1):
+            if row.id == team_id:
+                rank = i
+                break
+
+        # Return the rank if found, or a default rank otherwise
+        return rank if rank is not None else 0
+
+    # RIP scaled team ranking. Gone, but forgotten
+    # def get_team_rank_scaled(self, team_id: int) -> int:
+
+    def update_team(self, team_id: int, team_data: Dict) -> bool:
+        team_data["id"] = team_id
+
+        result = self.upsert(team, team_data)
+
+        if result is None:
+            self.logger.warn(
+                f"update_team: Failed to update team! team id: {team_id}"
+            )
+            return False
+        return True
+    def get_rival(self, rival_id: int) -> Optional[Row]:
+        sql = select(profile).where(profile.c.user == rival_id)
+        result = self.execute(sql)
+
+        if result is None:
+            return None
+        return result.fetchone()
+    def get_overview(self) -> Dict:
+        # Fetch and add up all the playcounts
+        playcount_sql = self.execute(select(profile.c.playCount))
+
+        if playcount_sql is None:
+            self.logger.warn(
+                f"get_overview: Couldn't pull playcounts"
+            )
+            return 0
+
+        total_play_count = 0;
+        for row in playcount_sql:
+            total_play_count += row[0]
+        return {
+            "total_play_count": total_play_count
+        }

titles/chuni/schema/score.py

@@ -4,9 +4,8 @@ from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteg
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.engine import Row
-from sqlalchemy.sql import func, select
-from sqlalchemy.dialects.mysql import insert
-
+from sqlalchemy.sql import func, select, insert
+from sqlalchemy.sql.expression import exists
 from core.data.schema import BaseData, metadata
 
 course = Table(
@@ -24,7 +23,7 @@ course = Table(
     Column("scoreMax", Integer),
     Column("isFullCombo", Boolean),
     Column("isAllJustice", Boolean),
-    Column("isSuccess", Boolean),
+    Column("isSuccess", Integer),
     Column("scoreRank", Integer),
     Column("eventId", Integer),
     Column("lastPlayDate", String(25)),
@@ -32,7 +31,7 @@ course = Table(
     Column("param2", Integer),
     Column("param3", Integer),
     Column("param4", Integer),
-    Column("isClear", Boolean),
+    Column("isClear", Integer),
     Column("theoryCount", Integer),
     Column("orderId", Integer),
     Column("playerRating", Integer),
@@ -60,7 +59,7 @@ best_score = Table(
     Column("maxComboCount", Integer),
     Column("isFullCombo", Boolean),
     Column("isAllJustice", Boolean),
-    Column("isSuccess", Boolean),
+    Column("isSuccess", Integer),
     Column("fullChain", Integer),
     Column("maxChain", Integer),
     Column("scoreRank", Integer),
@@ -125,7 +124,7 @@ playlog = Table(
     Column("characterId", Integer),
     Column("skillId", Integer),
     Column("playKind", Integer),
-    Column("isClear", Boolean),
+    Column("isClear", Integer),
     Column("skillLevel", Integer),
     Column("skillEffect", Integer),
     Column("placeName", String(255)),
@@ -134,7 +133,10 @@ playlog = Table(
     Column("charaIllustId", Integer),
     Column("romVersion", String(255)),
     Column("judgeHeaven", Integer),
-    mysql_charset="utf8mb4",
+    Column("regionId", Integer),
+    Column("machineType", Integer),
+    Column("ticketId", Integer),
+    mysql_charset="utf8mb4"
 )
 
 
@@ -151,10 +153,7 @@ class ChuniScoreData(BaseData):
         course_data["user"] = aime_id
         course_data = self.fix_bools(course_data)
 
-        sql = insert(course).values(**course_data)
-        conflict = sql.on_duplicate_key_update(**course_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(course, course_data)
         if result is None:
             return None
         return result.lastrowid
@@ -171,10 +170,8 @@ class ChuniScoreData(BaseData):
         score_data["user"] = aime_id
         score_data = self.fix_bools(score_data)
 
-        sql = insert(best_score).values(**score_data)
-        conflict = sql.on_duplicate_key_update(**score_data)
+        result = self.upsert(best_score, score_data)
 
-        result = self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
@@ -187,14 +184,66 @@ class ChuniScoreData(BaseData):
             return None
         return result.fetchall()
 
-    def put_playlog(self, aime_id: int, playlog_data: Dict) -> Optional[int]:
+    def put_playlog(self, aime_id: int, playlog_data: Dict, version: int) -> Optional[int]:
+        # Calculate the ROM version that should be inserted into the DB, based on the version of the ggame being inserted
+        # We only need from Version 10 (Plost) and back, as newer versions include romVersion in their upsert
+        # This matters both for gameRankings, as well as a future DB update to keep version data separate
+        romVer = {
+            10: "1.50.0",
+            9: "1.45.0",
+            8: "1.40.0",
+            7: "1.35.0",
+            6: "1.30.0",
+            5: "1.25.0",
+            4: "1.20.0",
+            3: "1.15.0",
+            2: "1.10.0",
+            1: "1.05.0",
+            0: "1.00.0"
+        }
+
         playlog_data["user"] = aime_id
         playlog_data = self.fix_bools(playlog_data)
+        if "romVersion" not in playlog_data:
+            playlog_data["romVersion"] = romVer.get(version, "1.00.0")
 
-        sql = insert(playlog).values(**playlog_data)
-        conflict = sql.on_duplicate_key_update(**playlog_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(playlog, playlog_data)
         if result is None:
             return None
         return result.lastrowid
+
+    def get_rankings(self, version: int) -> Optional[List[Dict]]:
+        # Calculates the ROM version that should be fetched for rankings, based on the game version being retrieved
+        # This prevents tracks that are not accessible in your version from counting towards the 10 results
+        romVer = {
+            13: "2.10%",
+            12: "2.05%",
+            11: "2.00%",
+            10: "1.50%",
+            9: "1.45%",
+            8: "1.40%",
+            7: "1.35%",
+            6: "1.30%",
+            5: "1.25%",
+            4: "1.20%",
+            3: "1.15%",
+            2: "1.10%",
+            1: "1.05%",
+            0: "1.00%"
+        }
+        sql = select([playlog.c.musicId.label('id'), func.count(playlog.c.musicId).label('point')]).where((playlog.c.level != 4) & (playlog.c.romVersion.like(romVer.get(version, "%")))).group_by(playlog.c.musicId).order_by(func.count(playlog.c.musicId).desc()).limit(10)
+        result = self.execute(sql)
+
+        if result is None:
+            return None
+
+        rows = result.fetchall()
+        return [dict(row) for row in rows]
+
+    def get_rival_music(self, rival_id: int) -> Optional[List[Dict]]:
+        sql = select(best_score).where(best_score.c.user == rival_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()

titles/chuni/schema/static.py

@@ -1,11 +1,18 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    ForeignKeyConstraint,
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.engine import Row
 from sqlalchemy.schema import ForeignKey
-from sqlalchemy.sql import func, select
-from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.sql import func, select, insert
+from datetime import datetime
 
 from core.data.schema import BaseData, metadata
 
@@ -17,6 +24,7 @@ events = Table(
     Column("eventId", Integer),
     Column("type", Integer),
     Column("name", String(255)),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
     Column("enabled", Boolean, server_default="1"),
     UniqueConstraint("version", "eventId", name="chuni_static_events_uk"),
     mysql_charset="utf8mb4",
@@ -125,11 +133,13 @@ gacha_cards = Table(
 login_bonus_preset = Table(
     "chuni_static_login_bonus_preset",
     metadata,
-    Column("id", Integer, primary_key=True, nullable=False),
+    Column("presetId", Integer, nullable=False),
     Column("version", Integer, nullable=False),
     Column("presetName", String(255), nullable=False),
     Column("isEnabled", Boolean, server_default="1"),
-    UniqueConstraint("version", "id", name="chuni_static_login_bonus_preset_uk"),
+    PrimaryKeyConstraint(
+        "presetId", "version", name="chuni_static_login_bonus_preset_pk"
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -138,15 +148,7 @@ login_bonus = Table(
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
     Column("version", Integer, nullable=False),
-    Column(
-        "presetId",
-        ForeignKey(
-            "chuni_static_login_bonus_preset.id",
-            ondelete="cascade",
-            onupdate="cascade",
-        ),
-        nullable=False,
-    ),
+    Column("presetId", Integer, nullable=False),
     Column("loginBonusId", Integer, nullable=False),
     Column("loginBonusName", String(255), nullable=False),
     Column("presentId", Integer, nullable=False),
@@ -157,6 +159,16 @@ login_bonus = Table(
     UniqueConstraint(
         "version", "presetId", "loginBonusId", name="chuni_static_login_bonus_uk"
     ),
+    ForeignKeyConstraint(
+        ["presetId", "version"],
+        [
+            "chuni_static_login_bonus_preset.presetId",
+            "chuni_static_login_bonus_preset.version",
+        ],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+        name="chuni_static_login_bonus_ibfk_1",
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -174,7 +186,7 @@ class ChuniStaticData(BaseData):
         need_login_day_count: int,
         login_bonus_category_type: int,
     ) -> Optional[int]:
-        sql = insert(login_bonus).values(
+        result = self.upsert(login_bonus, dict(
             version=version,
             presetId=preset_id,
             loginBonusId=login_bonus_id,
@@ -184,17 +196,7 @@ class ChuniStaticData(BaseData):
             itemNum=item_num,
             needLoginDayCount=need_login_day_count,
             loginBonusCategoryType=login_bonus_category_type,
-        )
-
-        conflict = sql.on_duplicate_key_update(
-            loginBonusName=login_bonus_name,
-            presentName=present_ame,
-            itemNum=item_num,
-            needLoginDayCount=need_login_day_count,
-            loginBonusCategoryType=login_bonus_category_type,
-        )
-
-        result = self.execute(conflict)
+        ))
         if result is None:
             return None
         return result.lastrowid
@@ -235,18 +237,12 @@ class ChuniStaticData(BaseData):
     def put_login_bonus_preset(
         self, version: int, preset_id: int, preset_name: str, is_enabled: bool
     ) -> Optional[int]:
-        sql = insert(login_bonus_preset).values(
-            id=preset_id,
+        result = self.upsert(login_bonus_preset, dict(
+            presetId=preset_id,
             version=version,
             presetName=preset_name,
             isEnabled=is_enabled,
-        )
-
-        conflict = sql.on_duplicate_key_update(
-            presetName=preset_name, isEnabled=is_enabled
-        )
-
-        result = self.execute(conflict)
+        ))
         if result is None:
             return None
         return result.lastrowid
@@ -269,13 +265,7 @@ class ChuniStaticData(BaseData):
     def put_event(
         self, version: int, event_id: int, type: int, name: str
     ) -> Optional[int]:
-        sql = insert(events).values(
-            version=version, eventId=event_id, type=type, name=name
-        )
-
-        conflict = sql.on_duplicate_key_update(name=name)
-
-        result = self.execute(conflict)
+        result = self.upsert(events, dict(version=version, eventId=event_id, type=type, name=name))
         if result is None:
             return None
         return result.lastrowid
@@ -289,14 +279,14 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(sql)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"update_event: failed to update event! version: {version}, event_id: {event_id}, enabled: {enabled}"
             )
             return None
 
         event = self.get_event(version, event_id)
         if event is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"update_event: failed to fetch event {event_id} after updating"
             )
             return None
@@ -342,7 +332,7 @@ class ChuniStaticData(BaseData):
         jacketPath: str,
         we_tag: str,
     ) -> Optional[int]:
-        sql = insert(music).values(
+        result = self.upsert(music, dict(
             version=version,
             songId=song_id,
             chartId=chart_id,
@@ -352,18 +342,7 @@ class ChuniStaticData(BaseData):
             genre=genre,
             jacketPath=jacketPath,
             worldsEndTag=we_tag,
-        )
-
-        conflict = sql.on_duplicate_key_update(
-            title=title,
-            artist=artist,
-            level=level,
-            genre=genre,
-            jacketPath=jacketPath,
-            worldsEndTag=we_tag,
-        )
-
-        result = self.execute(conflict)
+        ))
         if result is None:
             return None
         return result.lastrowid
@@ -377,23 +356,14 @@ class ChuniStaticData(BaseData):
         consume_type: int,
         selling_appeal: bool,
     ) -> Optional[int]:
-        sql = insert(charge).values(
+        result = self.upsert(charge, dict(
             version=version,
             chargeId=charge_id,
             name=name,
             expirationDays=expiration_days,
             consumeType=consume_type,
             sellingAppeal=selling_appeal,
-        )
-
-        conflict = sql.on_duplicate_key_update(
-            name=name,
-            expirationDays=expiration_days,
-            consumeType=consume_type,
-            sellingAppeal=selling_appeal,
-        )
-
-        result = self.execute(conflict)
+        ))
         if result is None:
             return None
         return result.lastrowid
@@ -416,6 +386,14 @@ class ChuniStaticData(BaseData):
             return None
         return result.fetchall()
 
+    def get_music(self, version: int) -> Optional[List[Row]]:
+        sql = music.select(music.c.version <= version)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_music_chart(
         self, version: int, song_id: int, chart_id: int
     ) -> Optional[List[Row]]:
@@ -432,6 +410,15 @@ class ChuniStaticData(BaseData):
             return None
         return result.fetchone()
 
+    def get_song(self, music_id: int) -> Optional[Row]:
+        sql = music.select(music.c.id == music_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+
     def put_avatar(
         self,
         version: int,
@@ -441,23 +428,14 @@ class ChuniStaticData(BaseData):
         iconPath: str,
         texturePath: str,
     ) -> Optional[int]:
-        sql = insert(avatar).values(
+        result = self.upsert(avatar, dict(
             version=version,
             avatarAccessoryId=avatarAccessoryId,
             name=name,
             category=category,
             iconPath=iconPath,
             texturePath=texturePath,
-        )
-
-        conflict = sql.on_duplicate_key_update(
-            name=name,
-            category=category,
-            iconPath=iconPath,
-            texturePath=texturePath,
-        )
-
-        result = self.execute(conflict)
+        ))
         if result is None:
             return None
         return result.lastrowid
@@ -469,23 +447,14 @@ class ChuniStaticData(BaseData):
         gacha_name: int,
         **gacha_data,
     ) -> Optional[int]:
-        sql = insert(gachas).values(
+        result = self.upsert(gachas, dict(
             version=version,
             gachaId=gacha_id,
             gachaName=gacha_name,
             **gacha_data,
-        )
-
-        conflict = sql.on_duplicate_key_update(
-            version=version,
-            gachaId=gacha_id,
-            gachaName=gacha_name,
-            **gacha_data,
-        )
-
-        result = self.execute(conflict)
+        ))
         if result is None:
-            self.logger.warn(f"Failed to insert gacha! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -512,15 +481,9 @@ class ChuniStaticData(BaseData):
     def put_gacha_card(
         self, gacha_id: int, card_id: int, **gacha_card
     ) -> Optional[int]:
-        sql = insert(gacha_cards).values(gachaId=gacha_id, cardId=card_id, **gacha_card)
-
-        conflict = sql.on_duplicate_key_update(
-            gachaId=gacha_id, cardId=card_id, **gacha_card
-        )
-
-        result = self.execute(conflict)
+        sql = self.upsert(gacha_cards, dict(gachaId=gacha_id, cardId=card_id, **gacha_card))
         if result is None:
-            self.logger.warn(f"Failed to insert gacha card! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha card! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -550,13 +513,13 @@ class ChuniStaticData(BaseData):
         return result.fetchone()
 
     def put_card(self, version: int, card_id: int, **card_data) -> Optional[int]:
-        sql = insert(cards).values(version=version, cardId=card_id, **card_data)
-
-        conflict = sql.on_duplicate_key_update(**card_data)
-
-        result = self.execute(conflict)
+        result = self.upsert(cards, dict(
+            version=version,
+            cardId=card_id,
+            **card_data,
+        ))
         if result is None:
-            self.logger.warn(f"Failed to insert card! card_id {card_id}")
+            self.logger.warning(f"Failed to insert card! card_id {card_id}")
             return None
         return result.lastrowid
 
@@ -566,4 +529,4 @@ class ChuniStaticData(BaseData):
         result = self.execute(sql)
         if result is None:
             return None
-        return result.fetchone()
+        return result.fetchone()

titles/chuni/sun.py

@@ -0,0 +1,37 @@
+from typing import Dict, Any
+
+from core.config import CoreConfig
+from titles.chuni.newplus import ChuniNewPlus
+from titles.chuni.const import ChuniConstants
+from titles.chuni.config import ChuniConfig
+
+
+class ChuniSun(ChuniNewPlus):
+    def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
+        super().__init__(core_cfg, game_cfg)
+        self.version = ChuniConstants.VER_CHUNITHM_SUN
+
+    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = super().handle_get_game_setting_api_request(data)
+        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)["rom"]
+        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)["data"]
+        ret["gameSetting"][
+            "matchingUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "matchingUriX"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "udpHolePunchUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "reflectorUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        return ret
+
+    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = super().handle_cm_get_user_preview_api_request(data)
+
+        # hardcode lastDataVersion for CardMaker 1.35 A032
+        user_data["lastDataVersion"] = "2.10.00"
+        return user_data

titles/chuni/sunplus.py

@@ -0,0 +1,37 @@
+from typing import Dict, Any
+
+from core.config import CoreConfig
+from titles.chuni.sun import ChuniSun
+from titles.chuni.const import ChuniConstants
+from titles.chuni.config import ChuniConfig
+
+
+class ChuniSunPlus(ChuniSun):
+    def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
+        super().__init__(core_cfg, game_cfg)
+        self.version = ChuniConstants.VER_CHUNITHM_SUN_PLUS
+
+    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = super().handle_get_game_setting_api_request(data)
+        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)["rom"]
+        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)["data"]
+        ret["gameSetting"][
+            "matchingUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/215/ChuniServlet/"
+        ret["gameSetting"][
+            "matchingUriX"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/215/ChuniServlet/"
+        ret["gameSetting"][
+            "udpHolePunchUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/215/ChuniServlet/"
+        ret["gameSetting"][
+            "reflectorUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/215/ChuniServlet/"
+        return ret
+
+    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = super().handle_cm_get_user_preview_api_request(data)
+
+        # I don't know if lastDataVersion is going to matter, I don't think CardMaker 1.35 works this far up
+        user_data["lastDataVersion"] = "2.15.00"
+        return user_data

titles/cm/base.py

@@ -4,7 +4,9 @@ import json
 import logging
 from enum import Enum
 
+import pytz
 from core.config import CoreConfig
+from core.utils import Utils
 from core.data.cache import cached
 from titles.cm.const import CardMakerConstants
 from titles.cm.config import CardMakerConfig
@@ -23,36 +25,77 @@ class CardMakerBase:
         self.game = CardMakerConstants.GAME_CODE
         self.version = CardMakerConstants.VER_CARD_MAKER
 
+    @staticmethod
+    def _parse_int_ver(version: str) -> str:
+        return version.replace(".", "")[:3]
+
     def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
-        if self.core_cfg.server.is_develop:
-            uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
+        if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
+            uri = f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}"
         else:
             uri = f"http://{self.core_cfg.title.hostname}"
 
-        # CHUNITHM = 0, maimai = 1, ONGEKI = 2
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "length": 3,
             "gameConnectList": [
-                {"modelKind": 0, "type": 1, "titleUri": f"{uri}/SDHD/200/"},
-                {"modelKind": 1, "type": 1, "titleUri": f"{uri}/SDEZ/120/"},
-                {"modelKind": 2, "type": 1, "titleUri": f"{uri}/SDDT/130/"},
+                # CHUNITHM
+                {
+                    "modelKind": 0,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDHD/{self._parse_int_ver(games_ver['chuni'])}/ChuniServlet/",
+                },
+                # maimai DX
+                {
+                    "modelKind": 1,
+                    "type": 1,
+                    "titleUri": f"{uri}/{self._parse_int_ver(games_ver['maimai'])}/Maimai2Servlet/",
+                },
+                # ONGEKI
+                {
+                    "modelKind": 2,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDDT/{self._parse_int_ver(games_ver['ongeki'])}/",
+                },
             ],
         }
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        reboot_start = date.strftime(
-            datetime.now() + timedelta(hours=3), self.date_time_format
-        )
-        reboot_end = date.strftime(
-            datetime.now() + timedelta(hours=4), self.date_time_format
-        )
+        # if reboot start/end time is not defined use the default behavior of being a few hours ago
+        if self.core_cfg.title.reboot_start_time == "" or self.core_cfg.title.reboot_end_time == "":
+            reboot_start = datetime.strftime(
+                datetime.utcnow() + timedelta(hours=6), self.date_time_format
+            )
+            reboot_end = datetime.strftime(
+                datetime.utcnow() + timedelta(hours=7), self.date_time_format
+            )
+        else:
+            # get current datetime in JST
+            current_jst = datetime.now(pytz.timezone('Asia/Tokyo')).date()
+
+            # parse config start/end times into datetime
+            reboot_start_time = datetime.strptime(self.core_cfg.title.reboot_start_time, "%H:%M")
+            reboot_end_time = datetime.strptime(self.core_cfg.title.reboot_end_time, "%H:%M")
+
+            # offset datetimes with current date/time
+            reboot_start_time = reboot_start_time.replace(year=current_jst.year, month=current_jst.month, day=current_jst.day, tzinfo=pytz.timezone('Asia/Tokyo'))
+            reboot_end_time = reboot_end_time.replace(year=current_jst.year, month=current_jst.month, day=current_jst.day, tzinfo=pytz.timezone('Asia/Tokyo'))
+
+            # create strings for use in gameSetting
+            reboot_start = reboot_start_time.strftime(self.date_time_format)
+            reboot_end = reboot_end_time.strftime(self.date_time_format)
+
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
 
         return {
             "gameSetting": {
                 "dataVersion": "1.30.00",
-                "ongekiCmVersion": "1.30.01",
-                "chuniCmVersion": "2.00.00",
-                "maimaiCmVersion": "1.20.00",
+                "ongekiCmVersion": games_ver["ongeki"],
+                "chuniCmVersion": games_ver["chuni"],
+                "maimaiCmVersion": games_ver["maimai"],
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,

titles/cm/cm135.py

@@ -1,8 +1,4 @@
-from datetime import date, datetime, timedelta
-from typing import Any, Dict, List
-import json
-import logging
-from enum import Enum
+from typing import Dict
 
 from core.config import CoreConfig
 from core.data.cache import cached
@@ -16,23 +12,7 @@ class CardMaker135(CardMakerBase):
         super().__init__(core_cfg, game_cfg)
         self.version = CardMakerConstants.VER_CARD_MAKER_135
 
-    def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_connect_api_request(data)
-        if self.core_cfg.server.is_develop:
-            uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
-        else:
-            uri = f"http://{self.core_cfg.title.hostname}"
-
-        ret["gameConnectList"][0]["titleUri"] = f"{uri}/SDHD/205/"
-        ret["gameConnectList"][1]["titleUri"] = f"{uri}/SDEZ/125/"
-        ret["gameConnectList"][2]["titleUri"] = f"{uri}/SDDT/135/"
-
-        return ret
-
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
-        ret["gameSetting"]["ongekiCmVersion"] = "1.35.03"
-        ret["gameSetting"]["chuniCmVersion"] = "2.05.00"
-        ret["gameSetting"]["maimaiCmVersion"] = "1.25.00"
         return ret

titles/cm/config.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from core.config import CoreConfig
 
 
@@ -20,6 +21,32 @@ class CardMakerServerConfig:
         )
 
 
+class CardMakerVersionConfig:
+    def __init__(self, parent_config: "CardMakerConfig") -> None:
+        self.__config = parent_config
+
+    def version(self, version: int) -> Dict:
+        """
+        in the form of:
+        1: {"ongeki": 1.30.01, "chuni": 2.00.00, "maimai": 1.20.00}
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "cardmaker", "version", default={
+                0: {    
+                    "ongeki": "1.30.01",
+                    "chuni": "2.00.00",
+                    "maimai": "1.20.00"
+                },
+                1: {
+                    "ongeki": "1.35.03",
+                    "chuni": "2.10.00",
+                    "maimai": "1.30.00"
+                }
+            }
+        )[version]
+
+
 class CardMakerConfig(dict):
     def __init__(self) -> None:
         self.server = CardMakerServerConfig(self)
+        self.version = CardMakerVersionConfig(self)

titles/cm/const.py

@@ -6,7 +6,7 @@ class CardMakerConstants:
     VER_CARD_MAKER = 0
     VER_CARD_MAKER_135 = 1
 
-    VERSION_NAMES = ("Card Maker 1.34", "Card Maker 1.35")
+    VERSION_NAMES = ("Card Maker 1.30", "Card Maker 1.35")
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/cm/index.py

@@ -7,21 +7,22 @@ import coloredlogs
 import zlib
 
 from os import path
-from typing import Tuple
+from typing import Tuple, List, Dict
 from twisted.web.http import Request
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
 from core.utils import Utils
-from titles.cm.config import CardMakerConfig
-from titles.cm.const import CardMakerConstants
-from titles.cm.base import CardMakerBase
-from titles.cm.cm135 import CardMaker135
+from core.title import BaseServlet
+from .config import CardMakerConfig
+from .const import CardMakerConstants
+from .base import CardMakerBase
+from .cm135 import CardMaker135
 
 
-class CardMakerServlet:
+class CardMakerServlet(BaseServlet):
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
-        self.core_cfg = core_cfg
+        super().__init__(core_cfg, cfg_dir)
         self.game_cfg = CardMakerConfig()
         if path.exists(f"{cfg_dir}/{CardMakerConstants.CONFIG_NAME}"):
             self.game_cfg.update(
@@ -30,7 +31,7 @@ class CardMakerServlet:
 
         self.versions = [
             CardMakerBase(core_cfg, self.game_cfg),
-            CardMaker135(core_cfg, self.game_cfg),
+            CardMaker135(core_cfg, self.game_cfg)
         ]
 
         self.logger = logging.getLogger("cardmaker")
@@ -55,11 +56,11 @@ class CardMakerServlet:
         coloredlogs.install(
             level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
         )
-
+    
     @classmethod
-    def get_allnet_info(
+    def is_game_enabled(
         cls, game_code: str, core_cfg: CoreConfig, cfg_dir: str
-    ) -> Tuple[bool, str, str]:
+    ) -> bool:
         game_cfg = CardMakerConfig()
         if path.exists(f"{cfg_dir}/{CardMakerConstants.CONFIG_NAME}"):
             game_cfg.update(
@@ -67,29 +68,26 @@ class CardMakerServlet:
             )
 
         if not game_cfg.server.enable:
-            return (False, "", "")
+            return False
 
-        if core_cfg.server.is_develop:
-            return (
-                True,
-                f"http://{core_cfg.title.hostname}:{core_cfg.title.port}/{game_code}/$v/",
-                "",
-            )
+        return True
+    
+    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
+        return (
+            [], 
+            [("render_POST", "/SDED/{version}/{endpoint}", {})]
+        )
 
-        return (True, f"http://{core_cfg.title.hostname}/{game_code}/$v/", "")
-
-    def render_POST(self, request: Request, version: int, url_path: str) -> bytes:
+    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+        version = int(matchers['version'])
+        endpoint = matchers['endpoint']
         req_raw = request.content.getvalue()
-        url_split = url_path.split("/")
         internal_ver = 0
-        endpoint = url_split[len(url_split) - 1]
         client_ip = Utils.get_ip_addr(request)
 
-        print(f"version: {version}")
-
         if version >= 130 and version < 135:  # Card Maker
             internal_ver = CardMakerConstants.VER_CARD_MAKER
-        elif version >= 135 and version < 136:  # Card Maker 1.35
+        elif version >= 135 and version < 140:  # Card Maker 1.35
             internal_ver = CardMakerConstants.VER_CARD_MAKER_135
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
@@ -124,11 +122,12 @@ class CardMakerServlet:
 
         except Exception as e:
             self.logger.error(f"Error handling v{version} method {endpoint} - {e}")
+            raise
             return zlib.compress(b'{"stat": "0"}')
 
         if resp is None:
             resp = {"returnCode": 1}
 
-        self.logger.info(f"Response {resp}")
+        self.logger.debug(f"Response {resp}")
 
         return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))

titles/cm/read.py

@@ -68,7 +68,7 @@ class CardMakerReader(BaseReader):
                     read_csv = getattr(CardMakerReader, func)
                     read_csv(self, f"{self.bin_dir}/MU3/{file}")
                 else:
-                    self.logger.warn(
+                    self.logger.warning(
                         f"Couldn't find {file} file in {self.bin_dir}, skipping"
                     )
 
@@ -89,8 +89,7 @@ class CardMakerReader(BaseReader):
         version_ids = {
             "v2_00": ChuniConstants.VER_CHUNITHM_NEW,
             "v2_05": ChuniConstants.VER_CHUNITHM_NEW_PLUS,
-            # Chunithm SUN, ignore for now
-            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1,
+            "v2_10": ChuniConstants.VER_CHUNITHM_SUN,
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -138,8 +137,7 @@ class CardMakerReader(BaseReader):
         version_ids = {
             "v2_00": ChuniConstants.VER_CHUNITHM_NEW,
             "v2_05": ChuniConstants.VER_CHUNITHM_NEW_PLUS,
-            # Chunithm SUN, ignore for now
-            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1,
+            "v2_10": ChuniConstants.VER_CHUNITHM_SUN,
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -207,6 +205,7 @@ class CardMakerReader(BaseReader):
             "1.20": Mai2Constants.VER_MAIMAI_DX_UNIVERSE,
             "1.25": Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS,
             "1.30": Mai2Constants.VER_MAIMAI_DX_FESTIVAL,
+            "1.35": Mai2Constants.VER_MAIMAI_DX_FESTIVAL_PLUS,
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -226,6 +225,12 @@ class CardMakerReader(BaseReader):
                             True if troot.find("disable").text == "false" else False
                         )
 
+                        # check if a date is part of the name and disable the
+                        # card if it is
+                        enabled = (
+                            False if re.search(r"\d{2}/\d{2}/\d{2}", name) else enabled
+                        )
+
                         self.mai2_data.static.put_card(
                             version, card_id, name, enabled=enabled
                         )

titles/cxb/base.py

@@ -2,15 +2,15 @@ import logging
 import json
 from decimal import Decimal
 from base64 import b64encode
-from typing import Any, Dict
-from hashlib import md5
-from datetime import datetime
+from typing import Any, Dict, List
+from os import path
 
 from core.config import CoreConfig
-from titles.cxb.config import CxbConfig
-from titles.cxb.const import CxbConstants
-from titles.cxb.database import CxbData
+from .config import CxbConfig
+from .const import CxbConstants
+from .database import CxbData
 
+from threading import Thread
 
 class CxbBase:
     def __init__(self, cfg: CoreConfig, game_cfg: CxbConfig) -> None:
@@ -21,6 +21,13 @@ class CxbBase:
         self.logger = logging.getLogger("cxb")
         self.version = CxbConstants.VER_CROSSBEATS_REV
 
+    def _get_data_contents(self, folder: str, filetype: str, encoding: str = None, subfolder: str = "") -> List[str]:
+        if path.exists(f"titles/cxb/data/{folder}/{subfolder}{filetype}.csv"):
+            with open(f"titles/cxb/data/{folder}/{subfolder}{filetype}.csv", encoding=encoding) as f:
+                return f.readlines()
+        
+        return []
+
     def handle_action_rpreq_request(self, data: Dict) -> Dict:
         return {}
 
@@ -51,9 +58,146 @@ class CxbBase:
             self.logger.info(f"Login user {data['login']['authid']}")
             return {"token": data["login"]["authid"], "uid": data["login"]["authid"]}
 
-        self.logger.warn(f"User {data['login']['authid']} does not have a profile")
+        self.logger.warning(f"User {data['login']['authid']} does not have a profile")
         return {}
 
+    def task_generateCoupon(index, data1):
+        # Coupons
+        for i in range(500, 510):
+            index.append(str(i))
+            couponid = int(i) - 500
+            dataValue = [
+                {
+                    "couponId": str(couponid),
+                    "couponNum": "1",
+                    "couponLog": [],
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateShopListTitle(index, data1):
+        # ShopList_Title
+        for i in range(200000, 201451):
+            index.append(str(i))
+            shopid = int(i) - 200000
+            dataValue = [
+                {
+                    "shopId": shopid,
+                    "shopState": "2",
+                    "isDisable": "t",
+                    "isDeleted": "f",
+                    "isSpecialFlag": "f",
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateShopListIcon(index, data1):
+        # ShopList_Icon
+        for i in range(202000, 202264):
+            index.append(str(i))
+            shopid = int(i) - 200000
+            dataValue = [
+                {
+                    "shopId": shopid,
+                    "shopState": "2",
+                    "isDisable": "t",
+                    "isDeleted": "f",
+                    "isSpecialFlag": "f",
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateStories(index, data1):
+        # Stories
+        for i in range(900000, 900003):
+            index.append(str(i))
+            storyid = int(i) - 900000
+            dataValue = [
+                {
+                    "storyId": storyid,
+                    "unlockState1": ["t"] * 10,
+                    "unlockState2": ["t"] * 10,
+                    "unlockState3": ["t"] * 10,
+                    "unlockState4": ["t"] * 10,
+                    "unlockState5": ["t"] * 10,
+                    "unlockState6": ["t"] * 10,
+                    "unlockState7": ["t"] * 10,
+                    "unlockState8": ["t"] * 10,
+                    "unlockState9": ["t"] * 10,
+                    "unlockState10": ["t"] * 10,
+                    "unlockState11": ["t"] * 10,
+                    "unlockState12": ["t"] * 10,
+                    "unlockState13": ["t"] * 10,
+                    "unlockState14": ["t"] * 10,
+                    "unlockState15": ["t"] * 10,
+                    "unlockState16": ["t"] * 10,
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateScoreData(song, index, data1):
+        song_data = song["data"]
+        songCode = []
+
+        songCode.append(
+            {
+                "mcode": song_data["mcode"],
+                "musicState": song_data["musicState"],
+                "playCount": song_data["playCount"],
+                "totalScore": song_data["totalScore"],
+                "highScore": song_data["highScore"],
+                "everHighScore": song_data["everHighScore"]
+                if "everHighScore" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "clearRate": song_data["clearRate"],
+                "rankPoint": song_data["rankPoint"],
+                "normalCR": song_data["normalCR"]
+                if "normalCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "survivalCR": song_data["survivalCR"]
+                if "survivalCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "ultimateCR": song_data["ultimateCR"]
+                if "ultimateCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "nohopeCR": song_data["nohopeCR"]
+                if "nohopeCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "combo": song_data["combo"],
+                "coupleUserId": song_data["coupleUserId"],
+                "difficulty": song_data["difficulty"],
+                "isFullCombo": song_data["isFullCombo"],
+                "clearGaugeType": song_data["clearGaugeType"],
+                "fieldType": song_data["fieldType"],
+                "gameType": song_data["gameType"],
+                "grade": song_data["grade"],
+                "unlockState": song_data["unlockState"],
+                "extraState": song_data["extraState"],
+            }
+        )
+        index.append(song_data["index"])
+        data1.append(
+            b64encode(
+                bytes(json.dumps(songCode[0], separators=(",", ":")), "utf-8")
+            ).decode("utf-8")
+        )
+
     def handle_action_loadrange_request(self, data: Dict) -> Dict:
         range_start = data["loadrange"]["range"][0]
         range_end = data["loadrange"]["range"][1]
@@ -107,145 +251,33 @@ class CxbBase:
         900000 = Stories
         """
 
-        # Coupons
-        for i in range(500, 510):
-            index.append(str(i))
-            couponid = int(i) - 500
-            dataValue = [
-                {
-                    "couponId": str(couponid),
-                    "couponNum": "1",
-                    "couponLog": [],
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+        # Async threads to generate the response
+        thread_Coupon = Thread(target=CxbBase.task_generateCoupon(index, data1))
+        thread_ShopListTitle = Thread(target=CxbBase.task_generateShopListTitle(index, data1))
+        thread_ShopListIcon = Thread(target=CxbBase.task_generateShopListIcon(index, data1))
+        thread_Stories = Thread(target=CxbBase.task_generateStories(index, data1))
 
-        # ShopList_Title
-        for i in range(200000, 201451):
-            index.append(str(i))
-            shopid = int(i) - 200000
-            dataValue = [
-                {
-                    "shopId": shopid,
-                    "shopState": "2",
-                    "isDisable": "t",
-                    "isDeleted": "f",
-                    "isSpecialFlag": "f",
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+        thread_Coupon.start()
+        thread_ShopListTitle.start()
+        thread_ShopListIcon.start()
+        thread_Stories.start()
 
-        # ShopList_Icon
-        for i in range(202000, 202264):
-            index.append(str(i))
-            shopid = int(i) - 200000
-            dataValue = [
-                {
-                    "shopId": shopid,
-                    "shopState": "2",
-                    "isDisable": "t",
-                    "isDeleted": "f",
-                    "isSpecialFlag": "f",
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
-
-        # Stories
-        for i in range(900000, 900003):
-            index.append(str(i))
-            storyid = int(i) - 900000
-            dataValue = [
-                {
-                    "storyId": storyid,
-                    "unlockState1": ["t"] * 10,
-                    "unlockState2": ["t"] * 10,
-                    "unlockState3": ["t"] * 10,
-                    "unlockState4": ["t"] * 10,
-                    "unlockState5": ["t"] * 10,
-                    "unlockState6": ["t"] * 10,
-                    "unlockState7": ["t"] * 10,
-                    "unlockState8": ["t"] * 10,
-                    "unlockState9": ["t"] * 10,
-                    "unlockState10": ["t"] * 10,
-                    "unlockState11": ["t"] * 10,
-                    "unlockState12": ["t"] * 10,
-                    "unlockState13": ["t"] * 10,
-                    "unlockState14": ["t"] * 10,
-                    "unlockState15": ["t"] * 10,
-                    "unlockState16": ["t"] * 10,
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+        thread_Coupon.join()
+        thread_ShopListTitle.join()
+        thread_ShopListIcon.join()
+        thread_Stories.join()
 
         for song in songs:
-            song_data = song["data"]
-            songCode = []
+            thread_ScoreData = Thread(target=CxbBase.task_generateScoreData(song, index, data1))
+            thread_ScoreData.start()
 
-            songCode.append(
-                {
-                    "mcode": song_data["mcode"],
-                    "musicState": song_data["musicState"],
-                    "playCount": song_data["playCount"],
-                    "totalScore": song_data["totalScore"],
-                    "highScore": song_data["highScore"],
-                    "everHighScore": song_data["everHighScore"]
-                    if "everHighScore" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "clearRate": song_data["clearRate"],
-                    "rankPoint": song_data["rankPoint"],
-                    "normalCR": song_data["normalCR"]
-                    if "normalCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "survivalCR": song_data["survivalCR"]
-                    if "survivalCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "ultimateCR": song_data["ultimateCR"]
-                    if "ultimateCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "nohopeCR": song_data["nohopeCR"]
-                    if "nohopeCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "combo": song_data["combo"],
-                    "coupleUserId": song_data["coupleUserId"],
-                    "difficulty": song_data["difficulty"],
-                    "isFullCombo": song_data["isFullCombo"],
-                    "clearGaugeType": song_data["clearGaugeType"],
-                    "fieldType": song_data["fieldType"],
-                    "gameType": song_data["gameType"],
-                    "grade": song_data["grade"],
-                    "unlockState": song_data["unlockState"],
-                    "extraState": song_data["extraState"],
-                }
-            )
-            index.append(song_data["index"])
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(songCode[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+        v_profile = self.data.profile.get_profile_index(0, uid, self.version)
+        v_profile_data = v_profile["data"]
 
-        for v in index:
-            try:
-                v_profile = self.data.profile.get_profile_index(0, uid, self.version)
-                v_profile_data = v_profile["data"]
+        for _, data in enumerate(profile):
+            if v_profile_data:
                 versionindex.append(int(v_profile_data["appVersion"]))
-            except:
+            else:
                 versionindex.append("10400")
 
         return {"index": index, "data": data1, "version": versionindex}
@@ -257,7 +289,7 @@ class CxbBase:
             # REV Omnimix Version Fetcher
             gameversion = data["saveindex"]["data"][0][2]
             self.logger.warning(f"Game Version is {gameversion}")
-        except:
+        except Exception:
             pass
 
         if "10205" in gameversion:
@@ -319,7 +351,7 @@ class CxbBase:
         # Sunrise
         try:
             profileIndex = save_data["index"].index("0")
-        except:
+        except Exception:
             return {"data": ""}  # Maybe
 
         profile = json.loads(save_data["data"][profileIndex])
@@ -416,7 +448,7 @@ class CxbBase:
         self.logger.info(f"Get best rankings for {uid}")
         p = self.data.score.get_best_rankings(uid)
 
-        rankList: list[Dict[str, Any]] = []
+        rankList: List[Dict[str, Any]] = []
 
         for rank in p:
             if rank["song_id"] is not None:
@@ -467,7 +499,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=rid["clear"],
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
@@ -485,7 +517,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=0,
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
@@ -501,7 +533,6 @@ class CxbBase:
         profile = self.data.profile.get_profile_index(0, uid, self.version)
         data1 = profile["data"]
         p = self.data.item.get_energy(uid)
-        energy = p["energy"]
 
         if not p:
             self.data.item.put_energy(uid, 5)
@@ -514,6 +545,7 @@ class CxbBase:
             }
 
         array = []
+        energy = p["energy"]
 
         newenergy = int(energy) + 5
         self.data.item.put_energy(uid, newenergy)
@@ -544,4 +576,4 @@ class CxbBase:
 
     def handle_action_stampreq_request(self, data: Dict) -> Dict:
         self.logger.info(data)
-        return {"stampreq": ""}
+        return {"stampreq": ""}

titles/cxb/config.py

@@ -19,42 +19,6 @@ class CxbServerConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "cxb", "server", "hostname", default="localhost"
-        )
-
-    @property
-    def ssl_enable(self) -> bool:
-        return CoreConfig.get_config_field(
-            self.__config, "cxb", "server", "ssl_enable", default=False
-        )
-
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "cxb", "server", "port", default=8082
-        )
-
-    @property
-    def port_secure(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "cxb", "server", "port_secure", default=443
-        )
-
-    @property
-    def ssl_cert(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "cxb", "server", "ssl_cert", default="cert/title.crt"
-        )
-
-    @property
-    def ssl_key(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "cxb", "server", "ssl_key", default="cert/title.key"
-        )
-
 
 class CxbConfig(dict):
     def __init__(self) -> None: