Merge remote-tracking branch 'origin/develop' into fork_develop

core/allnet.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Any, Optional, Tuple
+from typing import Dict, List, Any, Optional, Tuple, Union
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 from twisted.web.http import Request
@@ -11,6 +11,7 @@ from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
 from time import strptime
 from os import path
+import urllib.parse
 
 from core.config import CoreConfig
 from core.utils import Utils
@@ -79,7 +80,7 @@ class AllnetServlet:
             req = AllnetPowerOnRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
-            if not req.game_id or not req.ver or not req.serial or not req.ip:
+            if not req.game_id or not req.ver or not req.serial or not req.ip or not req.firm_ver or not req.boot_ver:
                 raise AllnetRequestException(
                     f"Bad auth request params from {request_ip} - {vars(req)}"
                 )
@@ -89,12 +90,14 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        if req.format_ver == "3":
+        if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
-        else:
+        elif req.format_ver == 2:
             resp = AllnetPowerOnResponse2()
+        else:
+            resp = AllnetPowerOnResponse()
 
-        self.logger.debug(f"Allnet request: {vars(req)}")
+        self.logger.debug(f"Allnet request: {vars(req)}")        
         if req.game_id not in self.uri_registry:
             if not self.config.server.is_develop:
                 msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
@@ -103,8 +106,9 @@ class AllnetServlet:
                 )
                 self.logger.warn(msg)
 
-                resp.stat = 0
-                return self.dict_to_http_form_string([vars(resp)])
+                resp.stat = -1
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
             else:
                 self.logger.info(
@@ -113,12 +117,15 @@ class AllnetServlet:
                 resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
                 resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
                 
-                self.logger.debug(f"Allnet response: {vars(resp)}")
-                return self.dict_to_http_form_string([vars(resp)])
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+                
+                self.logger.debug(f"Allnet response: {resp_str}")
+                return (resp_str + "\n").encode("utf-8")
 
         resp.uri, resp.host = self.uri_registry[req.game_id]
 
-        machine = self.data.arcade.get_machine(req.serial)
+        machine = self.data.arcade.get_machine(req.serial)        
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {req.serial} attempted allnet auth from {request_ip}."
             self.data.base.log_event(
@@ -126,8 +133,9 @@ class AllnetServlet:
             )
             self.logger.warn(msg)
 
-            resp.stat = 0
-            return self.dict_to_http_form_string([vars(resp)])
+            resp.stat = -2
+            resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
         if machine is not None:
             arcade = self.data.arcade.get_arcade(machine["arcade"])
@@ -169,9 +177,13 @@ class AllnetServlet:
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
         self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
-        self.logger.debug(f"Allnet response: {vars(resp)}")
 
-        return self.dict_to_http_form_string([vars(resp)]).encode("utf-8")
+        resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+        self.logger.debug(f"Allnet response: {resp_dict}")        
+        resp_str += "\n"
+
+        return resp_str.encode("utf-8")
 
     def handle_dlorder(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
@@ -196,13 +208,13 @@ class AllnetServlet:
         self.logger.info(
             f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
         )
-        resp = AllnetDownloadOrderResponse()
+        resp = AllnetDownloadOrderResponse(serial=req.serial)
 
         if (
             not self.config.allnet.allow_online_updates
             or not self.config.allnet.update_cfg_folder
         ):
-            return self.dict_to_http_form_string([vars(resp)])
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
 
         else:  # TODO: Keychip check
             if path.exists(
@@ -216,7 +228,9 @@ class AllnetServlet:
                 resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
 
             self.logger.debug(f"Sending download uri {resp.uri}")
-            return self.dict_to_http_form_string([vars(resp)])
+            self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
+
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
 
     def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
         if "file" not in match:
@@ -225,6 +239,8 @@ class AllnetServlet:
         req_file = match["file"].replace("%0A", "")
 
         if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
+            self.logger.info(f"Request for DL INI file {req_file} from {Utils.get_ip_addr(request)} successful")
+            self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
             return open(
                 f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
             ).read()
@@ -238,6 +254,27 @@ class AllnetServlet:
         )
         return b""
 
+    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
+        req_data = request.content.getvalue()
+        sections = req_data.decode("utf-8").split("\r\n")
+        
+        req_dict = dict(urllib.parse.parse_qsl(sections[0]))
+
+        serial: Union[str, None] = req_dict.get("serial", None)
+        num_files_to_dl: Union[str, None] = req_dict.get("nb_ftd", None)
+        num_files_dld: Union[str, None] = req_dict.get("nb_dld", None)
+        dl_state: Union[str, None] = req_dict.get("dld_st", None)
+        ip = Utils.get_ip_addr(request)
+
+        if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
+            return "NG".encode()
+
+        self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
+        return "OK".encode()
+    
+    def handle_alive(self, request: Request, match: Dict) -> bytes:
+        return "OK".encode()
+
     def handle_billing_request(self, request: Request, _: Dict):
         req_dict = self.billing_req_to_dict(request.content.getvalue())
         request_ip = Utils.get_ip_addr(request)
@@ -301,7 +338,7 @@ class AllnetServlet:
 
         resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
 
-        resp_str = self.dict_to_http_form_string([vars(resp)], True)
+        resp_str = self.dict_to_http_form_string([vars(resp)])
         if resp_str is None:
             self.logger.error(f"Failed to parse response {vars(resp)}")
 
@@ -312,21 +349,6 @@ class AllnetServlet:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
         return b"naomi ok"
 
-    def kvp_to_dict(self, kvp: List[str]) -> List[Dict[str, Any]]:
-        ret: List[Dict[str, Any]] = []
-        for x in kvp:
-            items = x.split("&")
-            tmp = {}
-
-            for item in items:
-                kvp = item.split("=")
-                if len(kvp) == 2:
-                    tmp[kvp[0]] = kvp[1]
-
-            ret.append(tmp)
-
-        return ret
-
     def billing_req_to_dict(self, data: bytes):
         """
         Parses an billing request string into a python dictionary
@@ -336,7 +358,10 @@ class AllnetServlet:
             unzipped = decomp.decompress(data)
             sections = unzipped.decode("ascii").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
@@ -351,7 +376,10 @@ class AllnetServlet:
             unzipped = zlib.decompress(zipped)
             sections = unzipped.decode("utf-8").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"allnet_req_to_dict: {e} while parsing {data}")
@@ -360,7 +388,7 @@ class AllnetServlet:
     def dict_to_http_form_string(
         self,
         data: List[Dict[str, Any]],
-        crlf: bool = False,
+        crlf: bool = True,
         trailing_newline: bool = True,
     ) -> Optional[str]:
         """
@@ -370,21 +398,19 @@ class AllnetServlet:
             urlencode = ""
             for item in data:
                 for k, v in item.items():
+                    if k is None or v is None:
+                        continue
                     urlencode += f"{k}={v}&"
-
                 if crlf:
                     urlencode = urlencode[:-1] + "\r\n"
                 else:
                     urlencode = urlencode[:-1] + "\n"
-
             if not trailing_newline:
                 if crlf:
                     urlencode = urlencode[:-2]
                 else:
                     urlencode = urlencode[:-1]
-
             return urlencode
-
         except Exception as e:
             self.logger.error(f"dict_to_http_form_string: {e} while parsing {data}")
             return None
@@ -394,20 +420,19 @@ class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
         if req is None:
             raise AllnetRequestException("Request processing failed")
-        self.game_id: str = req.get("game_id", "")
-        self.ver: str = req.get("ver", "")
-        self.serial: str = req.get("serial", "")
-        self.ip: str = req.get("ip", "")
-        self.firm_ver: str = req.get("firm_ver", "")
-        self.boot_ver: str = req.get("boot_ver", "")
-        self.encode: str = req.get("encode", "")
-        self.hops = int(req.get("hops", "0"))
-        self.format_ver = req.get("format_ver", "2")
-        self.token = int(req.get("token", "0"))
+        self.game_id: str = req.get("game_id", None)
+        self.ver: str = req.get("ver", None)
+        self.serial: str = req.get("serial", None)
+        self.ip: str = req.get("ip", None)
+        self.firm_ver: str = req.get("firm_ver", None)
+        self.boot_ver: str = req.get("boot_ver", None)
+        self.encode: str = req.get("encode", "EUC-JP")
+        self.hops = int(req.get("hops", "-1"))
+        self.format_ver = float(req.get("format_ver", "1.00"))
+        self.token: str = req.get("token", "0")
 
-
-class AllnetPowerOnResponse3:
-    def __init__(self, token) -> None:
+class AllnetPowerOnResponse:
+    def __init__(self) -> None:
         self.stat = 1
         self.uri = ""
         self.host = ""
@@ -418,40 +443,45 @@ class AllnetPowerOnResponse3:
         self.region_name0 = "W"
         self.region_name1 = ""
         self.region_name2 = ""
-        self.region_name3 = ""
-        self.country = "JPN"
-        self.allnet_id = "123"
-        self.client_timezone = "+0900"
-        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
-            "%Y-%m-%dT%H:%M:%SZ"
-        )
+        self.region_name3 = ""        
         self.setting = "1"
-        self.res_ver = "3"
-        self.token = str(token)
-
-
-class AllnetPowerOnResponse2:
-    def __init__(self) -> None:
-        self.stat = 1
-        self.uri = ""
-        self.host = ""
-        self.place_id = "123"
-        self.name = "ARTEMiS"
-        self.nickname = "ARTEMiS"
-        self.region0 = "1"
-        self.region_name0 = "W"
-        self.region_name1 = "X"
-        self.region_name2 = "Y"
-        self.region_name3 = "Z"
-        self.country = "JPN"
         self.year = datetime.now().year
         self.month = datetime.now().month
         self.day = datetime.now().day
         self.hour = datetime.now().hour
         self.minute = datetime.now().minute
         self.second = datetime.now().second
-        self.setting = "1"
-        self.timezone = "+0900"
+
+class AllnetPowerOnResponse3(AllnetPowerOnResponse):
+    def __init__(self, token) -> None:
+        super().__init__()
+
+        # Added in v3
+        self.country = "JPN"
+        self.allnet_id = "123"
+        self.client_timezone = "+0900"
+        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
+            "%Y-%m-%dT%H:%M:%SZ"
+        )
+        self.res_ver = "3"
+        self.token = token
+
+        # Removed in v3
+        self.year = None
+        self.month = None
+        self.day = None
+        self.hour = None
+        self.minute = None
+        self.second = None
+
+
+class AllnetPowerOnResponse2(AllnetPowerOnResponse):
+    def __init__(self) -> None:
+        super().__init__()
+
+        # Added in v2
+        self.country = "JPN"
+        self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
 

core/config.py

@@ -36,6 +36,12 @@ class ServerConfig:
             self.__config, "core", "server", "is_develop", default=True
         )
 
+    @property
+    def threading(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "threading", default=False
+        )
+
     @property
     def log_dir(self) -> str:
         return CoreConfig.get_config_field(

core/data/database.py

@@ -15,6 +15,13 @@ from core.utils import Utils
 
 
 class Data:
+    current_schema_version = 4
+    engine = None
+    session = None
+    user = None
+    arcade = None
+    card = None
+    base = None
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
 
@@ -24,22 +31,32 @@ class Data:
         else:
             self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
 
-        self.__engine = create_engine(self.__url, pool_recycle=3600)
-        session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
-        self.session = scoped_session(session)
+        if Data.engine is None:
+            Data.engine = create_engine(self.__url, pool_recycle=3600)
+            self.__engine = Data.engine
 
-        self.user = UserData(self.config, self.session)
-        self.arcade = ArcadeData(self.config, self.session)
-        self.card = CardData(self.config, self.session)
-        self.base = BaseData(self.config, self.session)
-        self.current_schema_version = 4
+        if Data.session is None:
+            s = sessionmaker(bind=Data.engine, autoflush=True, autocommit=True)
+            Data.session = scoped_session(s)
+
+        if Data.user is None:
+            Data.user = UserData(self.config, self.session)
+        
+        if Data.arcade is None:
+            Data.arcade = ArcadeData(self.config, self.session)
+        
+        if Data.card is None:
+            Data.card = CardData(self.config, self.session)
+        
+        if Data.base is None:
+            Data.base = BaseData(self.config, self.session)
 
-        log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("database")
 
         # Prevent the logger from adding handlers multiple times
-        if not getattr(self.logger, "handler_set", None):
+        if not getattr(self.logger, "handler_set", None):            
+            log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
             fileHandler = TimedRotatingFileHandler(
                 "{0}/{1}.log".format(self.config.server.log_dir, "db"),
                 encoding="utf-8",
@@ -333,3 +350,8 @@ class Data:
 
             if not failed:
                 self.base.set_schema_ver(latest_ver, game)
+    
+    def show_versions(self) -> None:
+        all_game_versions = self.base.get_all_schema_vers()
+        for ver in all_game_versions:
+            self.logger.info(f"{ver['game']} -> v{ver['version']}")

core/data/schema/versions/SDEZ_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP DEFAULT "2018-01-01 00:00:00.0",
+    CHANGE COLUMN endDate endDate TIMESTAMP DEFAULT "2038-01-01 00:00:00.0";

core/data/schema/versions/SDEZ_5_rollback.sql

@@ -0,0 +1,78 @@
+DELETE FROM mai2_static_event WHERE version < 13;
+UPDATE mai2_static_event SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_music WHERE version < 13;
+UPDATE mai2_static_music SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_ticket WHERE version < 13;
+UPDATE mai2_static_ticket SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_cards WHERE version < 13;
+UPDATE mai2_static_cards SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_detail WHERE version < 13;
+UPDATE mai2_profile_detail SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_extend WHERE version < 13;
+UPDATE mai2_profile_extend SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_option WHERE version < 13;
+UPDATE mai2_profile_option SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_ghost WHERE version < 13;
+UPDATE mai2_profile_ghost SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_rating WHERE version < 13;
+UPDATE mai2_profile_rating SET version = version - 13 WHERE version >= 13;
+
+DROP TABLE maimai_score_best;
+DROP TABLE maimai_playlog;
+DROP TABLE maimai_profile_detail;
+DROP TABLE maimai_profile_option;
+DROP TABLE maimai_profile_web_option;
+DROP TABLE maimai_profile_grade_status;
+
+ALTER TABLE mai2_item_character DROP COLUMN point;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NOT NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NOT NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NOT NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NOT NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NOT NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rank int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NOT NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;

core/data/schema/versions/SDEZ_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CHANGE COLUMN endDate endDate TIMESTAMP NOT NULL;

core/data/schema/versions/SDEZ_6_rollback.sql

@@ -0,0 +1 @@
+DROP TABLE aime.mai2_profile_consec_logins;

core/data/schema/versions/SDEZ_6_upgrade.sql

@@ -0,0 +1,62 @@
+UPDATE mai2_static_event SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_music SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_ticket SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_cards SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_detail  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_extend  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_option  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_ghost  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_rating  SET version = version + 13 WHERE version < 1000;
+
+ALTER TABLE mai2_item_character ADD point int(11) NULL;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rank int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;

core/data/schema/versions/SDEZ_7_upgrade.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `mai2_profile_consec_logins` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `logins` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `mai2_profile_consec_logins_uk` (`user`,`version`),
+  CONSTRAINT `mai2_profile_consec_logins_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

core/mucha.py

@@ -33,8 +33,8 @@ class MuchaServlet:
         self.logger.addHandler(fileHandler)
         self.logger.addHandler(consoleHandler)
 
-        self.logger.setLevel(logging.INFO)
-        coloredlogs.install(level=logging.INFO, logger=self.logger, fmt=log_fmt_str)
+        self.logger.setLevel(cfg.mucha.loglevel)
+        coloredlogs.install(level=cfg.mucha.loglevel, logger=self.logger, fmt=log_fmt_str)
 
         all_titles = Utils.get_all_titles()
 

core/title.py

@@ -84,7 +84,7 @@ class TitleServlet:
             request.setResponseCode(405)
             return b""
 
-        return index.render_GET(request, endpoints["version"], endpoints["endpoint"])
+        return index.render_GET(request, int(endpoints["version"]), endpoints["endpoint"])
 
     def render_POST(self, request: Request, endpoints: dict) -> bytes:
         code = endpoints["game"]

dbutils.py

@@ -84,5 +84,8 @@ if __name__ == "__main__":
 
     elif args.action == "cleanup":
         data.delete_hanging_users()
+    
+    elif args.action == "version":
+        data.show_versions()
 
     data.logger.info("Done")

docs/config.md

@@ -5,6 +5,7 @@
 - `allow_unregistered_serials`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
 - `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
 - `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
+- `threading`: Flags that `reactor.run` should be called via the `Thread` standard library. May provide a speed boost, but removes the ability to kill the server via `Ctrl + C`. Default: `False`
 - `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
 ## Title
 - `loglevel`: Logging level for the title server. Default `info`

docs/game_specific_info.md

@@ -127,28 +127,50 @@ Config file is located in `config/cxb.yaml`.
 
 ### SDEZ
 
-| Version ID | Version Name            |
-|------------|-------------------------|
-| 0          | maimai DX               |
-| 1          | maimai DX PLUS          |
-| 2          | maimai DX Splash        |
-| 3          | maimai DX Splash PLUS   |
-| 4          | maimai DX UNiVERSE      |
-| 5          | maimai DX UNiVERSE PLUS |
-| 6          | maimai DX FESTiVAL      |
+| Game Code | Version ID | Version Name            |
+|-----------|------------|-------------------------|
+
+
+For versions pre-dx
+| Game Code | Version ID | Version Name            |
+|-----------|------------|-------------------------|
+| SBXL      | 0          | maimai                  |
+| SBXL      | 1          | maimai PLUS             |
+| SBZF      | 2          | maimai GreeN            |
+| SBZF      | 3          | maimai GreeN PLUS       |
+| SDBM      | 4          | maimai ORANGE           |
+| SDBM      | 5          | maimai ORANGE PLUS      |
+| SDCQ      | 6          | maimai PiNK             |
+| SDCQ      | 7          | maimai PiNK PLUS        |
+| SDDK      | 8          | maimai MURASAKI         |
+| SDDK      | 9          | maimai MURASAKI PLUS    |
+| SDDZ      | 10         | maimai MILK             |
+| SDDZ      | 11         | maimai MILK PLUS        |
+| SDEY      | 12         | maimai FiNALE           |
+| SDEZ      | 13         | maimai DX               |
+| SDEZ      | 14         | maimai DX PLUS          |
+| SDEZ      | 15         | maimai DX Splash        |
+| SDEZ      | 16         | maimai DX Splash PLUS   |
+| SDEZ      | 17         | maimai DX Universe      |
+| SDEZ      | 18         | maimai DX Universe PLUS |
+| SDEZ      | 19         | maimai DX Festival      |
 
 ### Importer
 
 In order to use the importer locate your game installation folder and execute:
-
+DX:
 ```shell
-python read.py --series SDEZ --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --series <Game Code> --version <Version ID> --binfolder /path/to/StreamingAssets --optfolder /path/to/game/option/folder
+```
+Pre-DX:
+```shell
+python read.py --series <Game Code> --version <Version ID> --binfolder /path/to/data --optfolder /path/to/patch/data
 ```
-
 The importer for maimai DX will import Events, Music and Tickets.
 
-**NOTE: It is required to use the importer because the game will
-crash without Events!**
+The importer for maimai Pre-DX will import Events and Music. Not all games will have patch data. Milk - Finale have file encryption, and need an AES key. That key is not provided by the developers. For games that do use encryption, provide the key, as a hex string, with the `--extra` flag. Ex `--extra 00112233445566778899AABBCCDDEEFF`
+
+**Important: It is required to use the importer because some games may not function properly or even crash without Events!**
 
 ### Database upgrade
 
@@ -157,6 +179,7 @@ Always make sure your database (tables) are up-to-date, to do so go to the `core
 ```shell
 python dbutils.py --game SDEZ upgrade
 ```
+Pre-Dx uses the same database as DX, so only upgrade using the SDEZ game code!
 
 ## Hatsune Miku Project Diva
 
@@ -253,13 +276,13 @@ python dbutils.py --game SDDT upgrade
 
 | Version ID | Version Name    |
 |------------|-----------------|
-| 0          | Card Maker 1.34 |
+| 0          | Card Maker 1.30 |
 | 1          | Card Maker 1.35 |
 
 
 ### Support status
 
-* Card Maker 1.34:
+* Card Maker 1.30:
   * CHUNITHM NEW!!: Yes
   * maimai DX UNiVERSE: Yes
   * O.N.G.E.K.I. Bright: Yes
@@ -285,19 +308,46 @@ python read.py --series SDED --version <version ID> --binfolder titles/cm/cm_dat
 python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
-Also make sure to import all maimai and Chunithm data as well:
+Also make sure to import all maimai DX and CHUNITHM data as well:
 
 ```shell
 python read.py --series SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
 ```
 
-The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai/Chunithm) and the hardcoded
+The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai DX/CHUNITHM) and the hardcoded
 Cards for each Gacha (O.N.G.E.K.I. only).
 
 **NOTE: Without executing the importer Card Maker WILL NOT work!**
 
 
-### O.N.G.E.K.I. Gachas
+### Config setup
+
+Make sure to update your `config/cardmaker.yaml` with the correct version for each game. To get the current version required to run a specific game, open every opt (Axxx) folder descending until you find all three folders:
+
+- `MU3`: O.N.G.E.K.I.
+- `MAI`: maimai DX
+- `CHU`: CHUNITHM
+
+Inside each folder is a `DataConfig.xml` file, for example:
+
+`MU3/DataConfig.xml`:
+```xml
+  <cardMakerVersion>
+    <major>1</major>
+    <minor>35</minor>
+    <release>3</release>
+  </cardMakerVersion>
+```
+
+Now update your `config/cardmaker.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  1: # Card Maker 1.35
+    ongeki: 1.35.03
+```	 
+
+### O.N.G.E.K.I.
 
 Gacha "無料ガチャ" can only pull from the free cards with the following probabilities: 94%: R, 5% SR and 1% chance of
 getting an SSR card
@@ -310,20 +360,24 @@ and 3% chance of getting an SSR card
 All other (limited) gachas can pull from every card added to ongeki_static_cards but with the promoted cards
 (click on the green button under the banner) having a 10 times higher chance to get pulled
 
-### Chunithm Gachas
+### CHUNITHM
 
-All cards in Chunithm (basically just the characters) have the same rarity to it just pulls randomly from all cards
+All cards in CHUNITHM (basically just the characters) have the same rarity to it just pulls randomly from all cards
 from a given gacha but made sure you cannot pull the same card twice in the same 5 times gacha roll.
 
+### maimai DX
+
+Printed maimai DX cards: Freedom (`cardTypeId=6`) or Gold Pass (`cardTypeId=4`) can now be selected during the login process. You can only have ONE Freedom and ONE Gold Pass active at a given time. The cards will expire after 15 days.
+
+Thanks GetzeAvenue for the `selectedCardList` rarity hint!
+
 ### Notes
 
-Card Maker 1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35 will only load an O.N.G.E.K.I.
+Card Maker 1.30-1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35+ will only load an O.N.G.E.K.I.
 Bright Memory profile (1.35).
-The gachas inside the `ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
+The gachas inside the `config/ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
 Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded for CM 1.35.
 
-**NOTE: There is currently no way to load/use the (printed) maimai DX cards!**
-
 ## WACCA
 
 ### SDFE
@@ -367,6 +421,41 @@ Always make sure your database (tables) are up-to-date, to do so go to the `core
 python dbutils.py --game SDFE upgrade
 ```
 
+### VIP Rewards
+Below is a list of VIP rewards. Currently, VIP is not implemented, and thus these are not obtainable. These 23 rewards were distributed once per month for VIP users on the real network.
+
+	Plates:
+		211004 リッチ
+		211018 特盛えりざべす
+		211025 イースター
+		211026 特盛りりぃ
+		311004 ファンシー
+		311005 インカンテーション
+		311014 夜明け
+		311015 ネイビー
+		311016 特盛るーん
+	
+	Ring Colors:
+		203002 Gold Rushイエロー
+		203009 トロピカル
+		303005 ネイチャー
+	
+	Icons:
+		202020 どらみんぐ
+		202063 ユニコーン
+		202086 ゴリラ
+		302014 ローズ
+		302015 ファラオ
+		302045 肉球
+		302046 WACCA
+		302047 WACCA Lily
+		302048 WACCA Reverse
+	
+	Note Sound Effect:
+		205002 テニス
+		205008 シャワー
+		305003 タンバリンMk-Ⅱ
+
 ## SAO
 
 ### SDEW
@@ -405,6 +494,13 @@ Always make sure your database (tables) are up-to-date, to do so go to the `core
 python dbutils.py --game SDEW upgrade
 ```
 
+### Notes
+- Defrag Match will crash at loading
+- Co-Op Online is not supported
+- Shop is not functionnal
+- Player title is currently static and cannot be changed in-game
+- QR Card Scanning currently only load a static hero
+
 ### Credits for SAO support:
 
 - Midorica - Limited Network Support

example_config/cardmaker.yaml

@@ -1,3 +1,13 @@
 server:
   enable: True
   loglevel: "info"
+
+version:
+  0:
+    ongeki: 1.30.01
+    chuni: 2.00.00
+    maimai: 1.20.00
+  1:
+    ongeki: 1.35.03
+    chuni: 2.10.00
+    maimai: 1.30.00

example_config/core.yaml

@@ -4,6 +4,7 @@ server:
   allow_unregistered_serials: True
   name: "ARTEMiS"
   is_develop: True
+  threading: False
   log_dir: "logs"
 
 title:

example_config/mai2.yaml

@@ -1,3 +1,8 @@
 server:
   enable: True
   loglevel: "info"
+
+deliver:
+  enable: False
+  udbdl_enable: False
+  content_folder: ""

example_config/pokken.yaml

@@ -2,8 +2,11 @@ server:
   hostname: "localhost"
   enable: True
   loglevel: "info"
-  port: 9000
-  port_stun: 9001
-  port_turn: 9002
-  port_admission: 9003
-  auto_register: True
+  auto_register: True
+  enable_matching: False
+  stun_server_host: "stunserver.stunprotocol.org"
+  stun_server_port: 3478
+
+ports:
+  game: 9000
+  admission: 9001

index.py

@@ -11,7 +11,7 @@ from twisted.web import server, resource
 from twisted.internet import reactor, endpoints
 from twisted.web.http import Request
 from routes import Mapper
-
+from threading import Thread
 
 class HttpDispatcher(resource.Resource):
     def __init__(self, cfg: CoreConfig, config_dir: str):
@@ -63,6 +63,27 @@ class HttpDispatcher(resource.Resource):
             action="handle_dlorder",
             conditions=dict(method=["POST"]),
         )
+        self.map_post.connect(
+            "allnet_loaderstaterecorder",
+            "/sys/servlet/LoaderStateRecorder",
+            controller="allnet",
+            action="handle_loaderstaterecorder",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_get.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["GET"]),
+        )
         self.map_post.connect(
             "allnet_billing",
             "/request",
@@ -111,7 +132,6 @@ class HttpDispatcher(resource.Resource):
         )
 
     def render_GET(self, request: Request) -> bytes:
-        self.logger.debug(request.uri)
         test = self.map_get.match(request.uri.decode())
         client_ip = Utils.get_ip_addr(request)
 
@@ -161,9 +181,16 @@ class HttpDispatcher(resource.Resource):
 
         if type(ret) == str:
             return ret.encode()
-        elif type(ret) == bytes:
+        
+        elif type(ret) == bytes or type(ret) == tuple: # allow for bytes or tuple (data, response code) responses
             return ret
+        
+        elif ret is None:
+            self.logger.warn(f"None returned by controller for {request.uri.decode()} endpoint")
+            return b""
+        
         else:
+            self.logger.warn(f"Unknown data type returned by controller for {request.uri.decode()} endpoint")
             return b""
 
 
@@ -256,4 +283,7 @@ if __name__ == "__main__":
             server.Site(dispatcher)
         )
 
-    reactor.run()  # type: ignore
+    if cfg.server.threading:
+        Thread(target=reactor.run, args=(False,)).start()
+    else:
+        reactor.run()

readme.md

@@ -17,7 +17,7 @@ Games listed below have been tested and confirmed working. Only game versions ol
     + All versions
 
 + Card Maker
-    + 1.34
+    + 1.30
     + 1.35
 
 + O.N.G.E.K.I.
@@ -30,6 +30,9 @@ Games listed below have been tested and confirmed working. Only game versions ol
 + POKKÉN TOURNAMENT
     + Final Online
 
++ Sword Art Online Arcade (partial support)
+    + Final
+
 ## Requirements
 - python 3 (tested working with 3.9 and 3.10, other versions YMMV)
 - pip

requirements.txt

@@ -16,3 +16,4 @@ Routes
 bcrypt
 jinja2
 protobuf
+autobahn

titles/chuni/base.py

@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
 from time import strftime
 
 import pytz
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 from core.config import CoreConfig
 from titles.chuni.const import ChuniConstants
@@ -401,7 +401,7 @@ class ChuniBase:
                 "userItemList": [],
             }
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")

titles/chuni/schema/item.py

@@ -296,7 +296,7 @@ class ChuniItemData(BaseData):
         self,
         version: int,
         room_id: int,
-        matching_member_info_list: list,
+        matching_member_info_list: List,
         user_id: int = None,
         rest_sec: int = 60,
         is_full: bool = False

titles/cm/base.py

@@ -23,19 +23,40 @@ class CardMakerBase:
         self.game = CardMakerConstants.GAME_CODE
         self.version = CardMakerConstants.VER_CARD_MAKER
 
+    @staticmethod
+    def _parse_int_ver(version: str) -> str:
+        return version.replace(".", "")[:3]
+
     def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
         if self.core_cfg.server.is_develop:
             uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
         else:
             uri = f"http://{self.core_cfg.title.hostname}"
 
-        # CHUNITHM = 0, maimai = 1, ONGEKI = 2
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "length": 3,
             "gameConnectList": [
-                {"modelKind": 0, "type": 1, "titleUri": f"{uri}/SDHD/200/"},
-                {"modelKind": 1, "type": 1, "titleUri": f"{uri}/SDEZ/120/"},
-                {"modelKind": 2, "type": 1, "titleUri": f"{uri}/SDDT/130/"},
+                # CHUNITHM
+                {
+                    "modelKind": 0,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDHD/{self._parse_int_ver(games_ver['chuni'])}/",
+                },
+                # maimai DX
+                {
+                    "modelKind": 1,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDEZ/{self._parse_int_ver(games_ver['maimai'])}/",
+                },
+                # ONGEKI
+                {
+                    "modelKind": 2,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDDT/{self._parse_int_ver(games_ver['ongeki'])}/",
+                },
             ],
         }
 
@@ -47,12 +68,15 @@ class CardMakerBase:
             datetime.now() + timedelta(hours=4), self.date_time_format
         )
 
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "gameSetting": {
                 "dataVersion": "1.30.00",
-                "ongekiCmVersion": "1.30.01",
-                "chuniCmVersion": "2.00.00",
-                "maimaiCmVersion": "1.20.00",
+                "ongekiCmVersion": games_ver["ongeki"],
+                "chuniCmVersion": games_ver["chuni"],
+                "maimaiCmVersion": games_ver["maimai"],
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,

titles/cm/cm135.py

@@ -1,8 +1,4 @@
-from datetime import date, datetime, timedelta
-from typing import Any, Dict, List
-import json
-import logging
-from enum import Enum
+from typing import Dict
 
 from core.config import CoreConfig
 from core.data.cache import cached
@@ -16,23 +12,7 @@ class CardMaker135(CardMakerBase):
         super().__init__(core_cfg, game_cfg)
         self.version = CardMakerConstants.VER_CARD_MAKER_135
 
-    def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_connect_api_request(data)
-        if self.core_cfg.server.is_develop:
-            uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
-        else:
-            uri = f"http://{self.core_cfg.title.hostname}"
-
-        ret["gameConnectList"][0]["titleUri"] = f"{uri}/SDHD/205/"
-        ret["gameConnectList"][1]["titleUri"] = f"{uri}/SDEZ/125/"
-        ret["gameConnectList"][2]["titleUri"] = f"{uri}/SDDT/135/"
-
-        return ret
-
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
-        ret["gameSetting"]["ongekiCmVersion"] = "1.35.03"
-        ret["gameSetting"]["chuniCmVersion"] = "2.05.00"
-        ret["gameSetting"]["maimaiCmVersion"] = "1.25.00"
         return ret

titles/cm/config.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from core.config import CoreConfig
 
 
@@ -20,6 +21,21 @@ class CardMakerServerConfig:
         )
 
 
+class CardMakerVersionConfig:
+    def __init__(self, parent_config: "CardMakerConfig") -> None:
+        self.__config = parent_config
+
+    def version(self, version: int) -> Dict:
+        """
+        in the form of:
+        1: {"ongeki": 1.30.01, "chuni": 2.00.00, "maimai": 1.20.00}
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "cardmaker", "version", default={}
+        )[version]
+
+
 class CardMakerConfig(dict):
     def __init__(self) -> None:
         self.server = CardMakerServerConfig(self)
+        self.version = CardMakerVersionConfig(self)

titles/cm/const.py

@@ -6,7 +6,7 @@ class CardMakerConstants:
     VER_CARD_MAKER = 0
     VER_CARD_MAKER_135 = 1
 
-    VERSION_NAMES = ("Card Maker 1.34", "Card Maker 1.35")
+    VERSION_NAMES = ("Card Maker 1.30", "Card Maker 1.35")
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/cm/index.py

@@ -30,7 +30,7 @@ class CardMakerServlet:
 
         self.versions = [
             CardMakerBase(core_cfg, self.game_cfg),
-            CardMaker135(core_cfg, self.game_cfg),
+            CardMaker135(core_cfg, self.game_cfg)
         ]
 
         self.logger = logging.getLogger("cardmaker")
@@ -89,7 +89,7 @@ class CardMakerServlet:
 
         if version >= 130 and version < 135:  # Card Maker
             internal_ver = CardMakerConstants.VER_CARD_MAKER
-        elif version >= 135 and version < 136:  # Card Maker 1.35
+        elif version >= 135 and version < 140:  # Card Maker 1.35
             internal_ver = CardMakerConstants.VER_CARD_MAKER_135
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:

titles/cxb/base.py

@@ -2,7 +2,7 @@ import logging
 import json
 from decimal import Decimal
 from base64 import b64encode
-from typing import Any, Dict
+from typing import Any, Dict, List
 from hashlib import md5
 from datetime import datetime
 
@@ -416,7 +416,7 @@ class CxbBase:
         self.logger.info(f"Get best rankings for {uid}")
         p = self.data.score.get_best_rankings(uid)
 
-        rankList: list[Dict[str, Any]] = []
+        rankList: List[Dict[str, Any]] = []
 
         for rank in p:
             if rank["song_id"] is not None:

titles/cxb/index.py

@@ -103,7 +103,7 @@ class CxbServlet(resource.Resource):
             else:
                 self.logger.info(f"Ready on port {self.game_cfg.server.port}")
 
-    def render_POST(self, request: Request):
+    def render_POST(self, request: Request, version: int, endpoint: str):
         version = 0
         internal_ver = 0
         func_to_find = ""

titles/mai2/__init__.py

@@ -6,5 +6,14 @@ from titles.mai2.read import Mai2Reader
 index = Mai2Servlet
 database = Mai2Data
 reader = Mai2Reader
-game_codes = [Mai2Constants.GAME_CODE]
-current_schema_version = 4
+game_codes = [
+    Mai2Constants.GAME_CODE_DX,
+    Mai2Constants.GAME_CODE_FINALE,
+    Mai2Constants.GAME_CODE_MILK,
+    Mai2Constants.GAME_CODE_MURASAKI,
+    Mai2Constants.GAME_CODE_PINK,
+    Mai2Constants.GAME_CODE_ORANGE,
+    Mai2Constants.GAME_CODE_GREEN,
+    Mai2Constants.GAME_CODE,
+]
+current_schema_version = 7

titles/mai2/base.py

@@ -1,5 +1,5 @@
 from datetime import datetime, date, timedelta
-from typing import Any, Dict
+from typing import Any, Dict, List
 import logging
 
 from core.config import CoreConfig
@@ -12,40 +12,35 @@ class Mai2Base:
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         self.core_config = cfg
         self.game_config = game_cfg
-        self.game = Mai2Constants.GAME_CODE
-        self.version = Mai2Constants.VER_MAIMAI_DX
+        self.version = Mai2Constants.VER_MAIMAI
         self.data = Mai2Data(cfg)
         self.logger = logging.getLogger("mai2")
+        self.can_deliver = False
+        self.can_usbdl = False
+        self.old_server = ""
         
         if self.core_config.server.is_develop and self.core_config.title.port > 0:
-            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/100/"
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/197/"
         
         else:
-            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/100/"
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/197/"
 
     def handle_get_game_setting_api_request(self, data: Dict):
-        # TODO: See if making this epoch 0 breaks things
-        reboot_start = date.strftime(
-            datetime.now() + timedelta(hours=3), Mai2Constants.DATE_TIME_FORMAT
-        )
-        reboot_end = date.strftime(
-            datetime.now() + timedelta(hours=4), Mai2Constants.DATE_TIME_FORMAT
-        )
-        return {
+        return {            
+            "isDevelop": False,
+            "isAouAccession": False,
             "gameSetting": {
-                "isMaintenance": "false",
-                "requestInterval": 10,
-                "rebootStartTime": reboot_start,
-                "rebootEndTime": reboot_end,
-                "movieUploadLimit": 10000,
-                "movieStatus": 0,
-                "movieServerUri": "",
-                "deliverServerUri": "",
-                "oldServerUri": self.old_server,
-                "usbDlServerUri": "",
-                "rebootInterval": 0,
+                "isMaintenance": False,
+                "requestInterval": 1800,
+                "rebootStartTime": "2020-01-01 07:00:00.0",
+                "rebootEndTime": "2020-01-01 07:59:59.0",
+                "movieUploadLimit": 100,
+                "movieStatus": 1,
+                "movieServerUri": self.old_server + "api/movie" if self.game_config.uploads.movies else "movie",
+                "deliverServerUri": self.old_server + "deliver/" if self.can_deliver and self.game_config.deliver.enable else "",
+                "oldServerUri": self.old_server + "old",
+                "usbDlServerUri": self.old_server + "usbdl/" if self.can_deliver and self.game_config.deliver.udbdl_enable else "",
             },
-            "isAouAccession": "true",
         }
 
     def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
@@ -58,7 +53,7 @@ class Mai2Base:
     def handle_get_game_event_api_request(self, data: Dict) -> Dict:
         events = self.data.static.get_enabled_events(self.version)
         events_lst = []
-        if events is None:
+        if events is None or not events:
             self.logger.warn("No enabled events, did you run the reader?")
             return {"type": data["type"], "length": 0, "gameEventList": []}
 
@@ -117,43 +112,35 @@ class Mai2Base:
         return {"returnCode": 1, "apiName": "UpsertClientTestmodeApi"}
 
     def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_detail(data["userId"], self.version)
-        o = self.data.profile.get_profile_option(data["userId"], self.version)
-        if p is None or o is None:
+        p = self.data.profile.get_profile_detail(data["userId"], self.version, False)
+        w = self.data.profile.get_web_option(data["userId"], self.version)
+        if p is None or w is None:
             return {}  # Register
         profile = p._asdict()
-        option = o._asdict()
+        web_opt = w._asdict()
 
         return {
             "userId": data["userId"],
             "userName": profile["userName"],
             "isLogin": False,
-            "lastGameId": profile["lastGameId"],
             "lastDataVersion": profile["lastDataVersion"],
-            "lastRomVersion": profile["lastRomVersion"],
-            "lastLoginDate": profile["lastLoginDate"],
+            "lastLoginDate": profile["lastPlayDate"],
             "lastPlayDate": profile["lastPlayDate"],
             "playerRating": profile["playerRating"],
-            "nameplateId": 0,  # Unused
-            "iconId": profile["iconId"],
-            "trophyId": 0,  # Unused
-            "partnerId": profile["partnerId"],
+            "nameplateId": profile["nameplateId"],
             "frameId": profile["frameId"],
-            "dispRate": option[
-                "dispRate"
-            ],  # 0: all/begin, 1: disprate, 2: dispDan, 3: hide, 4: end
-            "totalAwake": profile["totalAwake"],
-            "isNetMember": profile["isNetMember"],
-            "dailyBonusDate": profile["dailyBonusDate"],
-            "headPhoneVolume": option["headPhoneVolume"],
-            "isInherit": False,  # Not sure what this is or does??
-            "banState": profile["banState"]
-            if profile["banState"] is not None
-            else 0,  # New with uni+
+            "iconId": profile["iconId"],
+            "trophyId": profile["trophyId"],
+            "dispRate": web_opt["dispRate"],  # 0: all, 1: dispRate, 2: dispDan, 3: hide
+            "dispRank": web_opt["dispRank"],
+            "dispHomeRanker": web_opt["dispHomeRanker"],
+            "dispTotalLv": web_opt["dispTotalLv"],
+            "totalLv": profile["totalLv"],
         }
 
     def handle_user_login_api_request(self, data: Dict) -> Dict:
         profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        consec = self.data.profile.get_consec_login(data["userId"], self.version)
 
         if profile is not None:
             lastLoginDate = profile["lastLoginDate"]
@@ -164,13 +151,32 @@ class Mai2Base:
         else:
             loginCt = 0
             lastLoginDate = "2017-12-05 07:00:00.0"
+        
+        if consec is None or not consec:
+            consec_ct = 1
+        
+        else:
+            lastlogindate_ = datetime.strptime(profile["lastLoginDate"], "%Y-%m-%d %H:%M:%S.%f").timestamp()
+            today_midnight = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp()
+            yesterday_midnight = today_midnight - 86400
+
+            if lastlogindate_ < today_midnight:
+                consec_ct = consec['logins'] + 1                
+                self.data.profile.add_consec_login(data["userId"], self.version)
+            
+            elif lastlogindate_ < yesterday_midnight:
+                consec_ct = 1
+                self.data.profile.reset_consec_login(data["userId"], self.version)
+
+            else:
+                consec_ct = consec['logins']
+        
 
         return {
             "returnCode": 1,
             "lastLoginDate": lastLoginDate,
             "loginCount": loginCt,
-            "consecutiveLoginCount": 0,  # We don't really have a way to track this...
-            "loginId": loginCt,  # Used with the playlog!
+            "consecutiveLoginCount": consec_ct, # Number of consecutive days we've logged in.
         }
 
     def handle_upload_user_playlog_api_request(self, data: Dict) -> Dict:
@@ -202,11 +208,34 @@ class Mai2Base:
         upsert = data["upsertUserAll"]
 
         if "userData" in upsert and len(upsert["userData"]) > 0:
-            upsert["userData"][0]["isNetMember"] = 1
             upsert["userData"][0].pop("accessCode")
+            upsert["userData"][0].pop("userId")
+
             self.data.profile.put_profile_detail(
-                user_id, self.version, upsert["userData"][0]
+                user_id, self.version, upsert["userData"][0], False
             )
+        
+        if "userWebOption" in upsert and len(upsert["userWebOption"]) > 0:            
+            upsert["userWebOption"][0]["isNetMember"] = True
+            self.data.profile.put_web_option(
+                user_id, self.version, upsert["userWebOption"][0]
+            )
+
+        if "userGradeStatusList" in upsert and len(upsert["userGradeStatusList"]) > 0:
+            self.data.profile.put_grade_status(
+                user_id, upsert["userGradeStatusList"][0]
+            )
+
+        if "userBossList" in upsert and len(upsert["userBossList"]) > 0:
+            self.data.profile.put_boss_list(
+                user_id, upsert["userBossList"][0]
+            )
+
+        if "userPlaylogList" in upsert and len(upsert["userPlaylogList"]) > 0:
+            for playlog in upsert["userPlaylogList"]:
+                self.data.score.put_playlog(
+                    user_id, playlog, False
+                )
 
         if "userExtend" in upsert and len(upsert["userExtend"]) > 0:
             self.data.profile.put_profile_extend(
@@ -215,11 +244,15 @@ class Mai2Base:
 
         if "userGhost" in upsert:
             for ghost in upsert["userGhost"]:
-                self.data.profile.put_profile_extend(user_id, self.version, ghost)
+                self.data.profile.put_profile_ghost(user_id, self.version, ghost)
+
+        if "userRecentRatingList" in upsert:
+            self.data.profile.put_recent_rating(user_id, upsert["userRecentRatingList"])
 
         if "userOption" in upsert and len(upsert["userOption"]) > 0:
+            upsert["userOption"][0].pop("userId")
             self.data.profile.put_profile_option(
-                user_id, self.version, upsert["userOption"][0]
+                user_id, self.version, upsert["userOption"][0], False
             )
 
         if "userRatingList" in upsert and len(upsert["userRatingList"]) > 0:
@@ -228,9 +261,8 @@ class Mai2Base:
             )
 
         if "userActivityList" in upsert and len(upsert["userActivityList"]) > 0:
-            for k, v in upsert["userActivityList"][0].items():
-                for act in v:
-                    self.data.profile.put_profile_activity(user_id, act)
+            for act in upsert["userActivityList"]:
+                self.data.profile.put_profile_activity(user_id, act)
 
         if "userChargeList" in upsert and len(upsert["userChargeList"]) > 0:
             for charge in upsert["userChargeList"]:
@@ -250,12 +282,9 @@ class Mai2Base:
 
         if "userCharacterList" in upsert and len(upsert["userCharacterList"]) > 0:
             for char in upsert["userCharacterList"]:
-                self.data.item.put_character(
+                self.data.item.put_character_(
                     user_id,
-                    char["characterId"],
-                    char["level"],
-                    char["awakening"],
-                    char["useCount"],
+                    char
                 )
 
         if "userItemList" in upsert and len(upsert["userItemList"]) > 0:
@@ -265,7 +294,7 @@ class Mai2Base:
                     int(item["itemKind"]),
                     item["itemId"],
                     item["stock"],
-                    item["isValid"],
+                    True
                 )
 
         if "userLoginBonusList" in upsert and len(upsert["userLoginBonusList"]) > 0:
@@ -291,7 +320,7 @@ class Mai2Base:
 
         if "userMusicDetailList" in upsert and len(upsert["userMusicDetailList"]) > 0:
             for music in upsert["userMusicDetailList"]:
-                self.data.score.put_best_score(user_id, music)
+                self.data.score.put_best_score(user_id, music, False)
 
         if "userCourseList" in upsert and len(upsert["userCourseList"]) > 0:
             for course in upsert["userCourseList"]:
@@ -319,7 +348,7 @@ class Mai2Base:
         return {"returnCode": 1}
 
     def handle_get_user_data_api_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        profile = self.data.profile.get_profile_detail(data["userId"], self.version, False)
         if profile is None:
             return
 
@@ -343,7 +372,7 @@ class Mai2Base:
         return {"userId": data["userId"], "userExtend": extend_dict}
 
     def handle_get_user_option_api_request(self, data: Dict) -> Dict:
-        options = self.data.profile.get_profile_option(data["userId"], self.version)
+        options = self.data.profile.get_profile_option(data["userId"], self.version, False)
         if options is None:
             return
 
@@ -413,12 +442,31 @@ class Mai2Base:
             "userChargeList": user_charge_list,
         }
 
+    def handle_get_user_present_api_request(self, data: Dict) -> Dict:
+        return { "userId": data.get("userId", 0), "length": 0, "userPresentList": []}
+    
+    def handle_get_transfer_friend_api_request(self, data: Dict) -> Dict:
+        return {}
+
+    def handle_get_user_present_event_api_request(self, data: Dict) -> Dict:
+        return { "userId": data.get("userId", 0), "length": 0, "userPresentEventList": []}
+    
+    def handle_get_user_boss_api_request(self, data: Dict) -> Dict:
+        b = self.data.profile.get_boss_list(data["userId"])
+        if b is None:
+            return { "userId": data.get("userId", 0), "userBossData": {}}
+        boss_lst = b._asdict()
+        boss_lst.pop("id")
+        boss_lst.pop("user")
+
+        return { "userId": data.get("userId", 0), "userBossData": boss_lst}
+
     def handle_get_user_item_api_request(self, data: Dict) -> Dict:
         kind = int(data["nextIndex"] / 10000000000)
         next_idx = int(data["nextIndex"] % 10000000000)
         user_item_list = self.data.item.get_items(data["userId"], kind)
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")
@@ -449,6 +497,8 @@ class Mai2Base:
             tmp = chara._asdict()
             tmp.pop("id")
             tmp.pop("user")
+            tmp.pop("awakening")
+            tmp.pop("useCount")
             chara_list.append(tmp)
 
         return {"userId": data["userId"], "userCharacterList": chara_list}
@@ -482,6 +532,16 @@ class Mai2Base:
 
         return {"userId": data["userId"], "userGhost": ghost_dict}
 
+    def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
+        rating = self.data.profile.get_recent_rating(data["userId"])
+        if rating is None:
+            return
+        
+        r = rating._asdict()
+        lst = r.get("userRecentRatingList", [])
+        
+        return {"userId": data["userId"], "length": len(lst), "userRecentRatingList": lst}
+
     def handle_get_user_rating_api_request(self, data: Dict) -> Dict:
         rating = self.data.profile.get_profile_rating(data["userId"], self.version)
         if rating is None:
@@ -644,25 +704,73 @@ class Mai2Base:
 
     def handle_get_user_region_api_request(self, data: Dict) -> Dict:
         return {"userId": data["userId"], "length": 0, "userRegionList": []}
+    
+    def handle_get_user_web_option_api_request(self, data: Dict) -> Dict:
+        w = self.data.profile.get_web_option(data["userId"], self.version)
+        if  w is None:
+            return {"userId": data["userId"], "userWebOption": {}}
+        
+        web_opt = w._asdict()        
+        web_opt.pop("id")
+        web_opt.pop("user")
+        web_opt.pop("version")
+
+        return {"userId": data["userId"], "userWebOption": web_opt}
+
+    def handle_get_user_survival_api_request(self, data: Dict) -> Dict:
+        return {"userId": data["userId"], "length": 0, "userSurvivalList": []}
+
+    def handle_get_user_grade_api_request(self, data: Dict) -> Dict:
+        g = self.data.profile.get_grade_status(data["userId"])
+        if g is None:
+            return {"userId": data["userId"], "userGradeStatus": {}, "length": 0, "userGradeList": []}
+        grade_stat = g._asdict()
+        grade_stat.pop("id")
+        grade_stat.pop("user")
+
+        return {"userId": data["userId"], "userGradeStatus": grade_stat, "length": 0, "userGradeList": []}
 
     def handle_get_user_music_api_request(self, data: Dict) -> Dict:
-        songs = self.data.score.get_best_scores(data["userId"])
+        user_id = data.get("userId", 0)        
+        next_index = data.get("nextIndex", 0)
+        max_ct = data.get("maxCount", 50)
+        upper_lim = next_index + max_ct
         music_detail_list = []
-        next_index = 0
 
-        if songs is not None:
-            for song in songs:
-                tmp = song._asdict()
-                tmp.pop("id")
-                tmp.pop("user")
-                music_detail_list.append(tmp)
+        if user_id <= 0:
+            self.logger.warn("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
+            return {}
+        
+        songs = self.data.score.get_best_scores(user_id, is_dx=False)
+        if songs is None:
+            self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
+            return {
+            "userId": data["userId"],
+            "nextIndex": 0,
+            "userMusicList": [],
+        }
 
-                if len(music_detail_list) == data["maxCount"]:
-                    next_index = data["maxCount"] + data["nextIndex"]
-                    break
+        num_user_songs = len(songs)
 
+        for x in range(next_index, upper_lim):
+            if num_user_songs <= x:
+                break
+
+            tmp = songs[x]._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            music_detail_list.append(tmp)
+
+        next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim
+        self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
         return {
             "userId": data["userId"],
             "nextIndex": next_index,
             "userMusicList": [{"userMusicDetailList": music_detail_list}],
         }
+
+    def handle_upload_user_portrait_api_request(self, data: Dict) -> Dict:
+        self.logger.debug(data)
+
+    def handle_upload_user_photo_api_request(self, data: Dict) -> Dict:
+        self.logger.debug(data)

titles/mai2/config.py

@@ -19,7 +19,59 @@ class Mai2ServerConfig:
             )
         )
 
+class Mai2DeliverConfig:
+    def __init__(self, parent: "Mai2Config") -> None:
+        self.__config = parent
+
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "deliver", "enable", default=False
+        )
+
+    @property
+    def udbdl_enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "deliver", "udbdl_enable", default=False
+        )
+    
+    @property
+    def content_folder(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "server", "content_folder", default=""
+        )
+
+class Mai2UploadsConfig:
+    def __init__(self, parent: "Mai2Config") -> None:
+        self.__config = parent
+    
+    @property
+    def photos(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "photos", default=False
+        )
+
+    @property
+    def photos_dir(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "photos_dir", default=""
+        )
+
+    @property
+    def movies(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "movies", default=False
+        )
+
+    @property
+    def movies_dir(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "movies_dir", default=""
+        )
+
 
 class Mai2Config(dict):
     def __init__(self) -> None:
         self.server = Mai2ServerConfig(self)
+        self.deliver = Mai2DeliverConfig(self)
+        self.uploads = Mai2UploadsConfig(self)

titles/mai2/const.py

@@ -20,19 +20,53 @@ class Mai2Constants:
 
     DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 
-    GAME_CODE = "SDEZ"
+    GAME_CODE = "SBXL"    
+    GAME_CODE_GREEN = "SBZF"    
+    GAME_CODE_ORANGE = "SDBM"    
+    GAME_CODE_PINK = "SDCQ"    
+    GAME_CODE_MURASAKI = "SDDK"    
+    GAME_CODE_MILK = "SDDZ"    
+    GAME_CODE_FINALE = "SDEY"    
+    GAME_CODE_DX = "SDEZ"
 
     CONFIG_NAME = "mai2.yaml"
 
-    VER_MAIMAI_DX = 0
-    VER_MAIMAI_DX_PLUS = 1
-    VER_MAIMAI_DX_SPLASH = 2
-    VER_MAIMAI_DX_SPLASH_PLUS = 3
-    VER_MAIMAI_DX_UNIVERSE = 4
-    VER_MAIMAI_DX_UNIVERSE_PLUS = 5
-    VER_MAIMAI_DX_FESTIVAL = 6
+    VER_MAIMAI = 0
+    VER_MAIMAI_PLUS = 1
+    VER_MAIMAI_GREEN = 2
+    VER_MAIMAI_GREEN_PLUS = 3
+    VER_MAIMAI_ORANGE = 4
+    VER_MAIMAI_ORANGE_PLUS = 5
+    VER_MAIMAI_PINK = 6
+    VER_MAIMAI_PINK_PLUS = 7
+    VER_MAIMAI_MURASAKI = 8
+    VER_MAIMAI_MURASAKI_PLUS = 9
+    VER_MAIMAI_MILK = 10
+    VER_MAIMAI_MILK_PLUS = 11
+    VER_MAIMAI_FINALE = 12
+
+    VER_MAIMAI_DX = 13
+    VER_MAIMAI_DX_PLUS = 14
+    VER_MAIMAI_DX_SPLASH = 15
+    VER_MAIMAI_DX_SPLASH_PLUS = 16
+    VER_MAIMAI_DX_UNIVERSE = 17
+    VER_MAIMAI_DX_UNIVERSE_PLUS = 18
+    VER_MAIMAI_DX_FESTIVAL = 19
 
     VERSION_STRING = (
+        "maimai",
+        "maimai PLUS",
+        "maimai GreeN",
+        "maimai GreeN PLUS",
+        "maimai ORANGE",
+        "maimai ORANGE PLUS",
+        "maimai PiNK",
+        "maimai PiNK PLUS",
+        "maimai MURASAKi",
+        "maimai MURASAKi PLUS",
+        "maimai MiLK",
+        "maimai MiLK PLUS",
+        "maimai FiNALE",        
         "maimai DX",
         "maimai DX PLUS",
         "maimai DX Splash",

titles/mai2/dx.py

@@ -0,0 +1,743 @@
+from typing import Any, List, Dict
+from datetime import datetime, timedelta
+import pytz
+import json
+from random import randint
+
+from core.config import CoreConfig
+from titles.mai2.base import Mai2Base
+from titles.mai2.config import Mai2Config
+from titles.mai2.const import Mai2Constants
+
+
+class Mai2DX(Mai2Base):
+    def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
+        super().__init__(cfg, game_cfg)
+        self.version = Mai2Constants.VER_MAIMAI_DX
+        
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEZ/100/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEZ/100/"
+    
+    def handle_get_game_setting_api_request(self, data: Dict):
+        return {
+            "gameSetting": {
+                "isMaintenance": False,
+                "requestInterval": 1800,
+                "rebootStartTime": "2020-01-01 07:00:00.0",
+                "rebootEndTime": "2020-01-01 07:59:59.0",
+                "movieUploadLimit": 100,
+                "movieStatus": 1,
+                "movieServerUri": self.old_server + "movie/",
+                "deliverServerUri": self.old_server + "deliver/" if self.can_deliver and self.game_config.deliver.enable else "",
+                "oldServerUri": self.old_server + "old",
+                "usbDlServerUri": self.old_server + "usbdl/" if self.can_deliver and self.game_config.deliver.udbdl_enable else "",
+                "rebootInterval": 0,
+            },
+            "isAouAccession": False,
+        }
+
+    def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
+        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+        o = self.data.profile.get_profile_option(data["userId"], self.version)
+        if p is None or o is None:
+            return {}  # Register
+        profile = p._asdict()
+        option = o._asdict()
+
+        return {
+            "userId": data["userId"],
+            "userName": profile["userName"],
+            "isLogin": False,
+            "lastGameId": profile["lastGameId"],
+            "lastDataVersion": profile["lastDataVersion"],
+            "lastRomVersion": profile["lastRomVersion"],
+            "lastLoginDate": profile["lastLoginDate"],
+            "lastPlayDate": profile["lastPlayDate"],
+            "playerRating": profile["playerRating"],
+            "nameplateId": 0,  # Unused
+            "iconId": profile["iconId"],
+            "trophyId": 0,  # Unused
+            "partnerId": profile["partnerId"],
+            "frameId": profile["frameId"],
+            "dispRate": option[
+                "dispRate"
+            ],  # 0: all/begin, 1: disprate, 2: dispDan, 3: hide, 4: end
+            "totalAwake": profile["totalAwake"],
+            "isNetMember": profile["isNetMember"],
+            "dailyBonusDate": profile["dailyBonusDate"],
+            "headPhoneVolume": option["headPhoneVolume"],
+            "isInherit": False,  # Not sure what this is or does??
+            "banState": profile["banState"]
+            if profile["banState"] is not None
+            else 0,  # New with uni+
+        }
+
+    def handle_upload_user_playlog_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        playlog = data["userPlaylog"]
+
+        self.data.score.put_playlog(user_id, playlog)
+
+        return {"returnCode": 1, "apiName": "UploadUserPlaylogApi"}
+
+    def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        charge = data["userCharge"]
+
+        # remove the ".0" from the date string, festival only?
+        charge["purchaseDate"] = charge["purchaseDate"].replace(".0", "")
+        self.data.item.put_charge(
+            user_id,
+            charge["chargeId"],
+            charge["stock"],
+            datetime.strptime(charge["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT),
+            datetime.strptime(charge["validDate"], Mai2Constants.DATE_TIME_FORMAT),
+        )
+
+        return {"returnCode": 1, "apiName": "UpsertUserChargelogApi"}
+
+    def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        upsert = data["upsertUserAll"]
+
+        if "userData" in upsert and len(upsert["userData"]) > 0:
+            upsert["userData"][0]["isNetMember"] = 1
+            upsert["userData"][0].pop("accessCode")
+            self.data.profile.put_profile_detail(
+                user_id, self.version, upsert["userData"][0]
+            )
+
+        if "userExtend" in upsert and len(upsert["userExtend"]) > 0:
+            self.data.profile.put_profile_extend(
+                user_id, self.version, upsert["userExtend"][0]
+            )
+
+        if "userGhost" in upsert:
+            for ghost in upsert["userGhost"]:
+                self.data.profile.put_profile_extend(user_id, self.version, ghost)
+
+        if "userOption" in upsert and len(upsert["userOption"]) > 0:
+            self.data.profile.put_profile_option(
+                user_id, self.version, upsert["userOption"][0]
+            )
+
+        if "userRatingList" in upsert and len(upsert["userRatingList"]) > 0:
+            self.data.profile.put_profile_rating(
+                user_id, self.version, upsert["userRatingList"][0]
+            )
+
+        if "userActivityList" in upsert and len(upsert["userActivityList"]) > 0:
+            for k, v in upsert["userActivityList"][0].items():
+                for act in v:
+                    self.data.profile.put_profile_activity(user_id, act)
+
+        if "userChargeList" in upsert and len(upsert["userChargeList"]) > 0:
+            for charge in upsert["userChargeList"]:
+                # remove the ".0" from the date string, festival only?
+                charge["purchaseDate"] = charge["purchaseDate"].replace(".0", "")
+                self.data.item.put_charge(
+                    user_id,
+                    charge["chargeId"],
+                    charge["stock"],
+                    datetime.strptime(
+                        charge["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT
+                    ),
+                    datetime.strptime(
+                        charge["validDate"], Mai2Constants.DATE_TIME_FORMAT
+                    ),
+                )
+
+        if "userCharacterList" in upsert and len(upsert["userCharacterList"]) > 0:
+            for char in upsert["userCharacterList"]:
+                self.data.item.put_character(
+                    user_id,
+                    char["characterId"],
+                    char["level"],
+                    char["awakening"],
+                    char["useCount"],
+                )
+
+        if "userItemList" in upsert and len(upsert["userItemList"]) > 0:
+            for item in upsert["userItemList"]:
+                self.data.item.put_item(
+                    user_id,
+                    int(item["itemKind"]),
+                    item["itemId"],
+                    item["stock"],
+                    item["isValid"],
+                )
+
+        if "userLoginBonusList" in upsert and len(upsert["userLoginBonusList"]) > 0:
+            for login_bonus in upsert["userLoginBonusList"]:
+                self.data.item.put_login_bonus(
+                    user_id,
+                    login_bonus["bonusId"],
+                    login_bonus["point"],
+                    login_bonus["isCurrent"],
+                    login_bonus["isComplete"],
+                )
+
+        if "userMapList" in upsert and len(upsert["userMapList"]) > 0:
+            for map in upsert["userMapList"]:
+                self.data.item.put_map(
+                    user_id,
+                    map["mapId"],
+                    map["distance"],
+                    map["isLock"],
+                    map["isClear"],
+                    map["isComplete"],
+                )
+
+        if "userMusicDetailList" in upsert and len(upsert["userMusicDetailList"]) > 0:
+            for music in upsert["userMusicDetailList"]:
+                self.data.score.put_best_score(user_id, music)
+
+        if "userCourseList" in upsert and len(upsert["userCourseList"]) > 0:
+            for course in upsert["userCourseList"]:
+                self.data.score.put_course(user_id, course)
+
+        if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0:
+            for fav in upsert["userFavoriteList"]:
+                self.data.item.put_favorite(user_id, fav["kind"], fav["itemIdList"])
+
+        if (
+            "userFriendSeasonRankingList" in upsert
+            and len(upsert["userFriendSeasonRankingList"]) > 0
+        ):
+            for fsr in upsert["userFriendSeasonRankingList"]:
+                fsr["recordDate"] = (
+                    datetime.strptime(
+                        fsr["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+                    ),
+                )
+                self.data.item.put_friend_season_ranking(user_id, fsr)
+
+        return {"returnCode": 1, "apiName": "UpsertUserAllApi"}
+
+    def handle_user_logout_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}
+
+    def handle_get_user_data_api_request(self, data: Dict) -> Dict:
+        profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        if profile is None:
+            return
+
+        profile_dict = profile._asdict()
+        profile_dict.pop("id")
+        profile_dict.pop("user")
+        profile_dict.pop("version")
+
+        return {"userId": data["userId"], "userData": profile_dict}
+
+    def handle_get_user_extend_api_request(self, data: Dict) -> Dict:
+        extend = self.data.profile.get_profile_extend(data["userId"], self.version)
+        if extend is None:
+            return
+
+        extend_dict = extend._asdict()
+        extend_dict.pop("id")
+        extend_dict.pop("user")
+        extend_dict.pop("version")
+
+        return {"userId": data["userId"], "userExtend": extend_dict}
+
+    def handle_get_user_option_api_request(self, data: Dict) -> Dict:
+        options = self.data.profile.get_profile_option(data["userId"], self.version)
+        if options is None:
+            return
+
+        options_dict = options._asdict()
+        options_dict.pop("id")
+        options_dict.pop("user")
+        options_dict.pop("version")
+
+        return {"userId": data["userId"], "userOption": options_dict}
+
+    def handle_get_user_card_api_request(self, data: Dict) -> Dict:
+        user_cards = self.data.item.get_cards(data["userId"])
+        if user_cards is None:
+            return {"userId": data["userId"], "nextIndex": 0, "userCardList": []}
+
+        max_ct = data["maxCount"]
+        next_idx = data["nextIndex"]
+        start_idx = next_idx
+        end_idx = max_ct + start_idx
+
+        if len(user_cards[start_idx:]) > max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        card_list = []
+        for card in user_cards:
+            tmp = card._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            tmp["startDate"] = datetime.strftime(
+                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["endDate"] = datetime.strftime(
+                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            card_list.append(tmp)
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userCardList": card_list[start_idx:end_idx],
+        }
+
+    def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
+        user_charges = self.data.item.get_charges(data["userId"])
+        if user_charges is None:
+            return {"userId": data["userId"], "length": 0, "userChargeList": []}
+
+        user_charge_list = []
+        for charge in user_charges:
+            tmp = charge._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            tmp["purchaseDate"] = datetime.strftime(
+                tmp["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["validDate"] = datetime.strftime(
+                tmp["validDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+
+            user_charge_list.append(tmp)
+
+        return {
+            "userId": data["userId"],
+            "length": len(user_charge_list),
+            "userChargeList": user_charge_list,
+        }
+
+    def handle_get_user_item_api_request(self, data: Dict) -> Dict:
+        kind = int(data["nextIndex"] / 10000000000)
+        next_idx = int(data["nextIndex"] % 10000000000)
+        user_item_list = self.data.item.get_items(data["userId"], kind)
+
+        items: List[Dict[str, Any]] = []
+        for i in range(next_idx, len(user_item_list)):
+            tmp = user_item_list[i]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            items.append(tmp)
+            if len(items) >= int(data["maxCount"]):
+                break
+
+        xout = kind * 10000000000 + next_idx + len(items)
+
+        if len(items) < int(data["maxCount"]):
+            next_idx = 0
+        else:
+            next_idx = xout
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "itemKind": kind,
+            "userItemList": items,
+        }
+
+    def handle_get_user_character_api_request(self, data: Dict) -> Dict:
+        characters = self.data.item.get_characters(data["userId"])
+
+        chara_list = []
+        for chara in characters:
+            tmp = chara._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            chara_list.append(tmp)
+
+        return {"userId": data["userId"], "userCharacterList": chara_list}
+
+    def handle_get_user_favorite_api_request(self, data: Dict) -> Dict:
+        favorites = self.data.item.get_favorites(data["userId"], data["itemKind"])
+        if favorites is None:
+            return
+
+        userFavs = []
+        for fav in favorites:
+            userFavs.append(
+                {
+                    "userId": data["userId"],
+                    "itemKind": fav["itemKind"],
+                    "itemIdList": fav["itemIdList"],
+                }
+            )
+
+        return {"userId": data["userId"], "userFavoriteData": userFavs}
+
+    def handle_get_user_ghost_api_request(self, data: Dict) -> Dict:
+        ghost = self.data.profile.get_profile_ghost(data["userId"], self.version)
+        if ghost is None:
+            return
+
+        ghost_dict = ghost._asdict()
+        ghost_dict.pop("user")
+        ghost_dict.pop("id")
+        ghost_dict.pop("version_int")
+
+        return {"userId": data["userId"], "userGhost": ghost_dict}
+
+    def handle_get_user_rating_api_request(self, data: Dict) -> Dict:
+        rating = self.data.profile.get_profile_rating(data["userId"], self.version)
+        if rating is None:
+            return
+
+        rating_dict = rating._asdict()
+        rating_dict.pop("user")
+        rating_dict.pop("id")
+        rating_dict.pop("version")
+
+        return {"userId": data["userId"], "userRating": rating_dict}
+
+    def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
+        """
+        kind 1 is playlist, kind 2 is music list
+        """
+        playlist = self.data.profile.get_profile_activity(data["userId"], 1)
+        musiclist = self.data.profile.get_profile_activity(data["userId"], 2)
+        if playlist is None or musiclist is None:
+            return
+
+        plst = []
+        mlst = []
+
+        for play in playlist:
+            tmp = play._asdict()
+            tmp["id"] = tmp["activityId"]
+            tmp.pop("activityId")
+            tmp.pop("user")
+            plst.append(tmp)
+
+        for music in musiclist:
+            tmp = music._asdict()
+            tmp["id"] = tmp["activityId"]
+            tmp.pop("activityId")
+            tmp.pop("user")
+            mlst.append(tmp)
+
+        return {"userActivity": {"playList": plst, "musicList": mlst}}
+
+    def handle_get_user_course_api_request(self, data: Dict) -> Dict:
+        user_courses = self.data.score.get_courses(data["userId"])
+        if user_courses is None:
+            return {"userId": data["userId"], "nextIndex": 0, "userCourseList": []}
+
+        course_list = []
+        for course in user_courses:
+            tmp = course._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            course_list.append(tmp)
+
+        return {"userId": data["userId"], "nextIndex": 0, "userCourseList": course_list}
+
+    def handle_get_user_portrait_api_request(self, data: Dict) -> Dict:
+        # No support for custom pfps
+        return {"length": 0, "userPortraitList": []}
+
+    def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict:
+        friend_season_ranking = self.data.item.get_friend_season_ranking(data["userId"])
+        if friend_season_ranking is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userFriendSeasonRankingList": [],
+            }
+
+        friend_season_ranking_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(friend_season_ranking)):
+            tmp = friend_season_ranking[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            tmp["recordDate"] = datetime.strftime(
+                tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+            )
+            friend_season_ranking_list.append(tmp)
+
+            if len(friend_season_ranking_list) >= max_ct:
+                break
+
+        if len(friend_season_ranking) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userFriendSeasonRankingList": friend_season_ranking_list,
+        }
+
+    def handle_get_user_map_api_request(self, data: Dict) -> Dict:
+        maps = self.data.item.get_maps(data["userId"])
+        if maps is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userMapList": [],
+            }
+
+        map_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(maps)):
+            tmp = maps[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            map_list.append(tmp)
+
+            if len(map_list) >= max_ct:
+                break
+
+        if len(maps) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userMapList": map_list,
+        }
+
+    def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
+        login_bonuses = self.data.item.get_login_bonuses(data["userId"])
+        if login_bonuses is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userLoginBonusList": [],
+            }
+
+        login_bonus_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(login_bonuses)):
+            tmp = login_bonuses[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            login_bonus_list.append(tmp)
+
+            if len(login_bonus_list) >= max_ct:
+                break
+
+        if len(login_bonuses) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userLoginBonusList": login_bonus_list,
+        }
+
+    def handle_get_user_region_api_request(self, data: Dict) -> Dict:
+        return {"userId": data["userId"], "length": 0, "userRegionList": []}
+
+    def handle_get_user_music_api_request(self, data: Dict) -> Dict:
+        songs = self.data.score.get_best_scores(data["userId"])
+        music_detail_list = []
+        next_index = 0
+
+        if songs is not None:
+            for song in songs:
+                tmp = song._asdict()
+                tmp.pop("id")
+                tmp.pop("user")
+                music_detail_list.append(tmp)
+
+                if len(music_detail_list) == data["maxCount"]:
+                    next_index = data["maxCount"] + data["nextIndex"]
+                    break
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_index,
+            "userMusicList": [{"userMusicDetailList": music_detail_list}],
+        }
+
+    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+        if p is None:
+            return {}
+
+        return {
+            "userName": p["userName"],
+            "rating": p["playerRating"],
+            # hardcode lastDataVersion for CardMaker 1.34
+            "lastDataVersion": "1.20.00",
+            "isLogin": False,
+            "isExistSellingCard": False,
+        }
+
+    def handle_cm_get_user_data_api_request(self, data: Dict) -> Dict:
+        # user already exists, because the preview checks that already
+        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+
+        cards = self.data.card.get_user_cards(data["userId"])
+        if cards is None or len(cards) == 0:
+            # This should never happen
+            self.logger.error(
+                f"handle_get_user_data_api_request: Internal error - No cards found for user id {data['userId']}"
+            )
+            return {}
+
+        # get the dict representation of the row so we can modify values
+        user_data = p._asdict()
+
+        # remove the values the game doesn't want
+        user_data.pop("id")
+        user_data.pop("user")
+        user_data.pop("version")
+
+        return {"userId": data["userId"], "userData": user_data}
+
+    def handle_cm_login_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}
+
+    def handle_cm_logout_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}
+
+    def handle_cm_get_selling_card_api_request(self, data: Dict) -> Dict:
+        selling_cards = self.data.static.get_enabled_cards(self.version)
+        if selling_cards is None:
+            return {"length": 0, "sellingCardList": []}
+
+        selling_card_list = []
+        for card in selling_cards:
+            tmp = card._asdict()
+            tmp.pop("id")
+            tmp.pop("version")
+            tmp.pop("cardName")
+            tmp.pop("enabled")
+
+            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["noticeStartDate"] = datetime.strftime(
+                tmp["noticeStartDate"], "%Y-%m-%d %H:%M:%S"
+            )
+            tmp["noticeEndDate"] = datetime.strftime(
+                tmp["noticeEndDate"], "%Y-%m-%d %H:%M:%S"
+            )
+
+            selling_card_list.append(tmp)
+
+        return {"length": len(selling_card_list), "sellingCardList": selling_card_list}
+
+    def handle_cm_get_user_card_api_request(self, data: Dict) -> Dict:
+        user_cards = self.data.item.get_cards(data["userId"])
+        if user_cards is None:
+            return {"returnCode": 1, "length": 0, "nextIndex": 0, "userCardList": []}
+
+        max_ct = data["maxCount"]
+        next_idx = data["nextIndex"]
+        start_idx = next_idx
+        end_idx = max_ct + start_idx
+
+        if len(user_cards[start_idx:]) > max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        card_list = []
+        for card in user_cards:
+            tmp = card._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+
+            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+            card_list.append(tmp)
+
+        return {
+            "returnCode": 1,
+            "length": len(card_list[start_idx:end_idx]),
+            "nextIndex": next_idx,
+            "userCardList": card_list[start_idx:end_idx],
+        }
+
+    def handle_cm_get_user_item_api_request(self, data: Dict) -> Dict:
+        super().handle_get_user_item_api_request(data)
+
+    def handle_cm_get_user_character_api_request(self, data: Dict) -> Dict:
+        characters = self.data.item.get_characters(data["userId"])
+
+        chara_list = []
+        for chara in characters:
+            chara_list.append(
+                {
+                    "characterId": chara["characterId"],
+                    # no clue why those values are even needed
+                    "point": 0,
+                    "count": 0,
+                    "level": chara["level"],
+                    "nextAwake": 0,
+                    "nextAwakePercent": 0,
+                    "favorite": False,
+                    "awakening": chara["awakening"],
+                    "useCount": chara["useCount"],
+                }
+            )
+
+        return {
+            "returnCode": 1,
+            "length": len(chara_list),
+            "userCharacterList": chara_list,
+        }
+
+    def handle_cm_get_user_card_print_error_api_request(self, data: Dict) -> Dict:
+        return {"length": 0, "userPrintDetailList": []}
+
+    def handle_cm_upsert_user_print_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        upsert = data["userPrintDetail"]
+
+        # set a random card serial number
+        serial_id = "".join([str(randint(0, 9)) for _ in range(20)])
+
+        user_card = upsert["userCard"]
+        self.data.item.put_card(
+            user_id,
+            user_card["cardId"],
+            user_card["cardTypeId"],
+            user_card["charaId"],
+            user_card["mapId"],
+        )
+
+        # properly format userPrintDetail for the database
+        upsert.pop("userCard")
+        upsert.pop("serialId")
+        upsert["printDate"] = datetime.strptime(upsert["printDate"], "%Y-%m-%d")
+
+        self.data.item.put_user_print_detail(user_id, serial_id, upsert)
+
+        return {
+            "returnCode": 1,
+            "orderId": 0,
+            "serialId": serial_id,
+            "startDate": "2018-01-01 00:00:00",
+            "endDate": "2038-01-01 00:00:00",
+        }
+
+    def handle_cm_upsert_user_printlog_api_request(self, data: Dict) -> Dict:
+        return {
+            "returnCode": 1,
+            "orderId": 0,
+            "serialId": data["userPrintlog"]["serialId"],
+        }
+
+    def handle_cm_upsert_buy_card_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}

titles/mai2/dxplus.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2Plus(Mai2Base):
+class Mai2DXPlus(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_PLUS

titles/mai2/festival.py

@@ -1,12 +1,12 @@
 from typing import Dict
 
 from core.config import CoreConfig
-from titles.mai2.universeplus import Mai2UniversePlus
+from titles.mai2.dx import Mai2DX
 from titles.mai2.const import Mai2Constants
 from titles.mai2.config import Mai2Config
 
 
-class Mai2Festival(Mai2UniversePlus):
+class Mai2Festival(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_FESTIVAL

titles/mai2/finale.py

@@ -0,0 +1,23 @@
+from typing import Any, List, Dict
+from datetime import datetime, timedelta
+import pytz
+import json
+
+from core.config import CoreConfig
+from titles.mai2.base import Mai2Base
+from titles.mai2.config import Mai2Config
+from titles.mai2.const import Mai2Constants
+
+
+class Mai2Finale(Mai2Base):
+    def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
+        super().__init__(cfg, game_cfg)
+        self.version = Mai2Constants.VER_MAIMAI_FINALE
+        self.can_deliver = True
+        self.can_usbdl = True
+        
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/197/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/197/"

titles/mai2/index.py

@@ -1,4 +1,5 @@
 from twisted.web.http import Request
+from twisted.web.server import NOT_DONE_YET
 import json
 import inflection
 import yaml
@@ -14,7 +15,9 @@ from core.utils import Utils
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 from titles.mai2.base import Mai2Base
-from titles.mai2.plus import Mai2Plus
+from titles.mai2.finale import Mai2Finale
+from titles.mai2.dx import Mai2DX
+from titles.mai2.dxplus import Mai2DXPlus
 from titles.mai2.splash import Mai2Splash
 from titles.mai2.splashplus import Mai2SplashPlus
 from titles.mai2.universe import Mai2Universe
@@ -33,7 +36,20 @@ class Mai2Servlet:
 
         self.versions = [
             Mai2Base,
-            Mai2Plus,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            Mai2Finale,            
+            Mai2DX,
+            Mai2DXPlus,
             Mai2Splash,
             Mai2SplashPlus,
             Mai2Universe,
@@ -42,27 +58,29 @@ class Mai2Servlet:
         ]
 
         self.logger = logging.getLogger("mai2")
-        log_fmt_str = "[%(asctime)s] Mai2 | %(levelname)s | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.core_cfg.server.log_dir, "mai2"),
-            encoding="utf8",
-            when="d",
-            backupCount=10,
-        )
+        if not hasattr(self.logger, "initted"):
+            log_fmt_str = "[%(asctime)s] Mai2 | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.core_cfg.server.log_dir, "mai2"),
+                encoding="utf8",
+                when="d",
+                backupCount=10,
+            )
 
-        fileHandler.setFormatter(log_fmt)
+            fileHandler.setFormatter(log_fmt)
 
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
 
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
 
-        self.logger.setLevel(self.game_cfg.server.loglevel)
-        coloredlogs.install(
-            level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
+            self.logger.setLevel(self.game_cfg.server.loglevel)
+            coloredlogs.install(
+                level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initted = True
 
     @classmethod
     def get_allnet_info(
@@ -82,7 +100,7 @@ class Mai2Servlet:
             return (
                 True,
                 f"http://{core_cfg.title.hostname}:{core_cfg.title.port}/{game_code}/$v/",
-                f"{core_cfg.title.hostname}:{core_cfg.title.port}",
+                f"{core_cfg.title.hostname}",
             )
 
         return (
@@ -94,6 +112,10 @@ class Mai2Servlet:
     def render_POST(self, request: Request, version: int, url_path: str) -> bytes:
         if url_path.lower() == "ping":
             return zlib.compress(b'{"returnCode": "1"}')
+        
+        elif url_path.startswith("api/movie/"):
+            self.logger.info(f"Movie data: {url_path} - {request.content.getvalue()}")
+            return b""
 
         req_raw = request.content.getvalue()
         url = request.uri.decode()
@@ -102,21 +124,50 @@ class Mai2Servlet:
         endpoint = url_split[len(url_split) - 1]
         client_ip = Utils.get_ip_addr(request)
 
-        if version < 105:  # 1.0
-            internal_ver = Mai2Constants.VER_MAIMAI_DX
-        elif version >= 105 and version < 110:  # Plus
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_PLUS
-        elif version >= 110 and version < 115:  # Splash
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH
-        elif version >= 115 and version < 120:  # Splash Plus
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS
-        elif version >= 120 and version < 125:  # Universe
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
-        elif version >= 125 and version < 130:  # Universe Plus
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS
-        elif version >= 130:  # Festival
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL
+        if request.uri.startswith(b"/SDEZ"):
+            if version < 105:  # 1.0
+                internal_ver = Mai2Constants.VER_MAIMAI_DX
+            elif version >= 105 and version < 110:  # Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_PLUS
+            elif version >= 110 and version < 115:  # Splash
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH
+            elif version >= 115 and version < 120:  # Splash Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS
+            elif version >= 120 and version < 125:  # Universe
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
+            elif version >= 125 and version < 130:  # Universe Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS
+            elif version >= 130:  # Festival
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL
 
+        else:
+            if version < 110:  # 1.0
+                internal_ver = Mai2Constants.VER_MAIMAI
+            elif version >= 110 and version < 120:  # Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_PLUS
+            elif version >= 120 and version < 130:  # Green
+                internal_ver = Mai2Constants.VER_MAIMAI_GREEN
+            elif version >= 130 and version < 140:  # Green Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_GREEN_PLUS
+            elif version >= 140 and version < 150:  # Orange
+                internal_ver = Mai2Constants.VER_MAIMAI_ORANGE
+            elif version >= 150 and version < 160:  # Orange Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_ORANGE_PLUS
+            elif version >= 160 and version < 170:  # Pink
+                internal_ver = Mai2Constants.VER_MAIMAI_PINK
+            elif version >= 170 and version < 180:  # Pink Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_PINK_PLUS
+            elif version >= 180 and version < 185:  # Murasaki
+                internal_ver = Mai2Constants.VER_MAIMAI_MURASAKI
+            elif version >= 185 and version < 190:  # Murasaki Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_MURASAKI_PLUS
+            elif version >= 190 and version < 195:  # Milk
+                internal_ver = Mai2Constants.VER_MAIMAI_MILK
+            elif version >= 195 and version < 197:  # Milk Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_MILK_PLUS
+            elif version >= 197:  # Finale
+                internal_ver = Mai2Constants.VER_MAIMAI_FINALE
+        
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
             # If we get a 32 character long hex string, it's a hash and we're
             # doing encrypted. The likelyhood of false positives is low but
@@ -159,3 +210,39 @@ class Mai2Servlet:
         self.logger.debug(f"Response {resp}")
 
         return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))
+
+    def render_GET(self, request: Request, version: int, url_path: str) -> bytes:
+        self.logger.info(f"v{version} GET {url_path}")
+        url_split = url_path.split("/")
+
+        if (url_split[0] == "api" and url_split[1] == "movie") or url_split[0] == "movie":
+            if url_split[2] == "moviestart":
+                return json.dumps({"moviestart":{"status":"OK"}}).encode()
+
+        if url_split[0] == "old":
+            if url_split[1] == "ping":
+                self.logger.info(f"v{version} old server ping")
+                return zlib.compress(b"ok")
+
+            elif url_split[1].startswith("userdata"):
+                self.logger.info(f"v{version} old server userdata inquire")
+                return zlib.compress(b"{}")
+
+            elif url_split[1].startswith("friend"):
+                self.logger.info(f"v{version} old server friend inquire")
+                return zlib.compress(b"{}")
+        
+        elif url_split[0] == "usbdl":
+            if url_split[1] == "CONNECTIONTEST":
+                self.logger.info(f"v{version} usbdl server test")
+                return zlib.compress(b"ok")
+
+        elif url_split[0] == "deliver":
+            file = url_split[len(url_split) - 1]
+            self.logger.info(f"v{version} {file} deliver inquire")
+            
+            if not self.game_cfg.deliver.enable or not path.exists(f"{self.game_cfg.deliver.content_folder}/{file}"):
+                return zlib.compress(b"")
+        
+        else:
+            return zlib.compress(b"{}")

titles/mai2/read.py

@@ -4,6 +4,9 @@ import os
 import re
 import xml.etree.ElementTree as ET
 from typing import Any, Dict, List, Optional
+from Crypto.Cipher import AES
+import zlib
+import codecs
 
 from core.config import CoreConfig
 from core.data import Data
@@ -34,18 +37,147 @@ class Mai2Reader(BaseReader):
 
     def read(self) -> None:
         data_dirs = []
-        if self.bin_dir is not None:
-            data_dirs += self.get_data_directories(self.bin_dir)
+        if self.version >= Mai2Constants.VER_MAIMAI_DX:
+            if self.bin_dir is not None:
+                data_dirs += self.get_data_directories(self.bin_dir)
 
-        if self.opt_dir is not None:
-            data_dirs += self.get_data_directories(self.opt_dir)
+            if self.opt_dir is not None:
+                data_dirs += self.get_data_directories(self.opt_dir)
 
-        for dir in data_dirs:
-            self.logger.info(f"Read from {dir}")
-            self.get_events(f"{dir}/event")
-            self.disable_events(f"{dir}/information", f"{dir}/scoreRanking")
-            self.read_music(f"{dir}/music")
-            self.read_tickets(f"{dir}/ticket")
+            for dir in data_dirs:
+                self.logger.info(f"Read from {dir}")
+                self.get_events(f"{dir}/event")
+                self.disable_events(f"{dir}/information", f"{dir}/scoreRanking")
+                self.read_music(f"{dir}/music")
+                self.read_tickets(f"{dir}/ticket")
+        
+        else:
+            if not os.path.exists(f"{self.bin_dir}/tables"):
+                self.logger.error(f"tables directory not found in {self.bin_dir}")
+                return
+            
+            if self.version >= Mai2Constants.VER_MAIMAI_MILK:
+                if self.extra is None:
+                    self.logger.error("Milk - Finale requre an AES key via a hex string send as the --extra flag")
+                    return
+                
+                key = bytes.fromhex(self.extra)
+            
+            else:
+                key = None
+            
+            evt_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmEvent.bin", key)
+            txt_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmtextout_jp.bin", key)
+            score_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmScore.bin", key)
+            
+            self.read_old_events(evt_table)
+            self.read_old_music(score_table, txt_table)
+            
+            if self.opt_dir is not None:
+                evt_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmEvent.bin", key)
+                txt_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmtextout_jp.bin", key)
+                score_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmScore.bin", key)
+
+                self.read_old_events(evt_table)
+                self.read_old_music(score_table, txt_table)
+
+            return
+    
+    def load_table_raw(self, dir: str, file: str, key: Optional[bytes]) -> Optional[List[Dict[str, str]]]:
+        if not os.path.exists(f"{dir}/{file}"):
+            self.logger.warn(f"file {file} does not exist in directory {dir}, skipping")
+            return
+        
+        self.logger.info(f"Load table {file} from {dir}")
+        if key is not None:
+            cipher = AES.new(key, AES.MODE_CBC)
+            with open(f"{dir}/{file}", "rb") as f:
+                f_encrypted = f.read()
+                f_data = cipher.decrypt(f_encrypted)[0x10:]
+        
+        else:
+            with open(f"{dir}/{file}", "rb") as f:
+                f_data = f.read()[0x10:]
+        
+        if f_data is None or not f_data:
+            self.logger.warn(f"file {dir} could not be read, skipping")
+            return
+        
+        f_data_deflate = zlib.decompress(f_data, wbits = zlib.MAX_WBITS | 16)[0x12:] # lop off the junk at the beginning
+        f_decoded = codecs.utf_16_le_decode(f_data_deflate)[0]
+        f_split = f_decoded.splitlines()
+
+        has_struct_def = "struct " in f_decoded
+        is_struct = False
+        struct_def = []
+        tbl_content = []
+
+        if has_struct_def:
+            for x in f_split:
+                if x.startswith("struct "):
+                    is_struct = True
+                    struct_name = x[7:-1]
+                    continue
+                
+                if x.startswith("};"):
+                    is_struct = False
+                    break
+
+                if is_struct:
+                    try:
+                        struct_def.append(x[x.rindex("  ") + 2: -1])
+                    except ValueError:
+                        self.logger.warn(f"rindex failed on line {x}")
+            
+            if is_struct:
+                self.logger.warn("Struct not formatted properly")
+            
+            if not struct_def:
+                self.logger.warn("Struct def not found")
+                    
+        name = file[:file.index(".")]
+        if "_" in name:
+            name = name[:file.index("_")]
+        
+        for x in f_split:
+            if not x.startswith(name.upper()):
+                continue
+            
+            line_match = re.match(r"(\w+)\((.*?)\)([ ]+\/{3}<[ ]+(.*))?", x)
+            if line_match is None:
+                continue
+
+            if not line_match.group(1) == name.upper():
+                self.logger.warn(f"Strange regex match for line {x} -> {line_match}")
+                continue
+            
+            vals = line_match.group(2)
+            comment = line_match.group(4)
+            line_dict = {}
+            
+            vals_split = vals.split(",")
+            for y in range(len(vals_split)):
+                stripped = vals_split[y].strip().lstrip("L\"").lstrip("\"").rstrip("\"")
+                if not stripped or stripped is None:
+                    continue
+
+                if has_struct_def and len(struct_def) > y:
+                    line_dict[struct_def[y]] = stripped
+                
+                else:
+                    line_dict[f'item_{y}'] = stripped
+            
+            if comment:
+                line_dict['comment'] = comment
+            
+            tbl_content.append(line_dict)
+
+        if tbl_content:
+            return tbl_content
+        
+        else:
+            self.logger.warning("Failed load table content, skipping")
+            return
 
     def get_events(self, base_dir: str) -> None:
         self.logger.info(f"Reading events from {base_dir}...")
@@ -188,3 +320,24 @@ class Mai2Reader(BaseReader):
                             self.version, id, ticket_type, price, name
                         )
                         self.logger.info(f"Added ticket {id}...")
+
+    def read_old_events(self, events: Optional[List[Dict[str, str]]]) -> None:
+        if events is None:
+            return
+        
+        for event in events:
+            evt_id = int(event.get('イベントID', '0'))
+            evt_expire_time = float(event.get('オフ時強制時期', '0.0'))
+            is_exp = bool(int(event.get('海外許可', '0')))
+            is_aou = bool(int(event.get('AOU許可', '0')))
+            name = event.get('comment', f'evt_{evt_id}')
+
+            self.data.static.put_game_event(self.version, 0, evt_id, name)
+            
+            if not (is_exp or is_aou):
+                self.data.static.toggle_game_event(self.version, evt_id, False)
+    
+    def read_old_music(self, scores: Optional[List[Dict[str, str]]], text: Optional[List[Dict[str, str]]]) -> None:
+        if scores is None or text is None:
+            return
+        # TODO

titles/mai2/schema/item.py

@@ -18,10 +18,11 @@ character = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("characterId", Integer, nullable=False),
-    Column("level", Integer, nullable=False, server_default="1"),
-    Column("awakening", Integer, nullable=False, server_default="0"),
-    Column("useCount", Integer, nullable=False, server_default="0"),
+    Column("characterId", Integer),
+    Column("level", Integer),
+    Column("awakening", Integer),
+    Column("useCount", Integer),
+    Column("point", Integer),
     UniqueConstraint("user", "characterId", name="mai2_item_character_uk"),
     mysql_charset="utf8mb4",
 )
@@ -35,12 +36,12 @@ card = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("cardId", Integer, nullable=False),
-    Column("cardTypeId", Integer, nullable=False),
-    Column("charaId", Integer, nullable=False),
-    Column("mapId", Integer, nullable=False),
-    Column("startDate", TIMESTAMP, server_default="2018-01-01 00:00:00.0"),
-    Column("endDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"),
+    Column("cardId", Integer),
+    Column("cardTypeId", Integer),
+    Column("charaId", Integer),
+    Column("mapId", Integer),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
+    Column("endDate", TIMESTAMP),
     UniqueConstraint("user", "cardId", "cardTypeId", name="mai2_item_card_uk"),
     mysql_charset="utf8mb4",
 )
@@ -54,10 +55,10 @@ item = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("itemId", Integer, nullable=False),
-    Column("itemKind", Integer, nullable=False),
-    Column("stock", Integer, nullable=False, server_default="1"),
-    Column("isValid", Boolean, nullable=False, server_default="1"),
+    Column("itemId", Integer),
+    Column("itemKind", Integer),
+    Column("stock", Integer),
+    Column("isValid", Boolean),
     UniqueConstraint("user", "itemId", "itemKind", name="mai2_item_item_uk"),
     mysql_charset="utf8mb4",
 )
@@ -71,11 +72,11 @@ map = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("mapId", Integer, nullable=False),
-    Column("distance", Integer, nullable=False),
-    Column("isLock", Boolean, nullable=False, server_default="0"),
-    Column("isClear", Boolean, nullable=False, server_default="0"),
-    Column("isComplete", Boolean, nullable=False, server_default="0"),
+    Column("mapId", Integer),
+    Column("distance", Integer),
+    Column("isLock", Boolean),
+    Column("isClear", Boolean),
+    Column("isComplete", Boolean),
     UniqueConstraint("user", "mapId", name="mai2_item_map_uk"),
     mysql_charset="utf8mb4",
 )
@@ -89,10 +90,10 @@ login_bonus = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("bonusId", Integer, nullable=False),
-    Column("point", Integer, nullable=False),
-    Column("isCurrent", Boolean, nullable=False, server_default="0"),
-    Column("isComplete", Boolean, nullable=False, server_default="0"),
+    Column("bonusId", Integer),
+    Column("point", Integer),
+    Column("isCurrent", Boolean),
+    Column("isComplete", Boolean),
     UniqueConstraint("user", "bonusId", name="mai2_item_login_bonus_uk"),
     mysql_charset="utf8mb4",
 )
@@ -106,12 +107,12 @@ friend_season_ranking = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("seasonId", Integer, nullable=False),
-    Column("point", Integer, nullable=False),
-    Column("rank", Integer, nullable=False),
-    Column("rewardGet", Boolean, nullable=False),
-    Column("userName", String(8), nullable=False),
-    Column("recordDate", TIMESTAMP, nullable=False),
+    Column("seasonId", Integer),
+    Column("point", Integer),
+    Column("rank", Integer),
+    Column("rewardGet", Boolean),
+    Column("userName", String(8)),
+    Column("recordDate", TIMESTAMP),
     UniqueConstraint(
         "user", "seasonId", "userName", name="mai2_item_friend_season_ranking_uk"
     ),
@@ -127,7 +128,7 @@ favorite = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("itemKind", Integer, nullable=False),
+    Column("itemKind", Integer),
     Column("itemIdList", JSON),
     UniqueConstraint("user", "itemKind", name="mai2_item_favorite_uk"),
     mysql_charset="utf8mb4",
@@ -142,10 +143,10 @@ charge = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("chargeId", Integer, nullable=False),
-    Column("stock", Integer, nullable=False),
-    Column("purchaseDate", String(255), nullable=False),
-    Column("validDate", String(255), nullable=False),
+    Column("chargeId", Integer),
+    Column("stock", Integer),
+    Column("purchaseDate", String(255)),
+    Column("validDate", String(255)),
     UniqueConstraint("user", "chargeId", name="mai2_item_charge_uk"),
     mysql_charset="utf8mb4",
 )
@@ -161,11 +162,11 @@ print_detail = Table(
     ),
     Column("orderId", Integer),
     Column("printNumber", Integer),
-    Column("printDate", TIMESTAMP, nullable=False, server_default=func.now()),
-    Column("serialId", String(20), nullable=False),
-    Column("placeId", Integer, nullable=False),
-    Column("clientId", String(11), nullable=False),
-    Column("printerSerialId", String(20), nullable=False),
+    Column("printDate", TIMESTAMP, server_default=func.now()),
+    Column("serialId", String(20)),
+    Column("placeId", Integer),
+    Column("clientId", String(11)),
+    Column("printerSerialId", String(20)),
     Column("cardRomVersion", Integer),
     Column("isHolograph", Boolean, server_default="1"),
     Column("printOption1", Boolean, server_default="0"),
@@ -332,6 +333,19 @@ class Mai2ItemData(BaseData):
         if result is None:
             return None
         return result.fetchone()
+    
+    def put_character_(self, user_id: int, char_data: Dict) -> Optional[int]:
+        char_data["user"] = user_id
+        sql = insert(character).values(**char_data)
+
+        conflict = sql.on_duplicate_key_update(**char_data)
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_character_: failed to insert item! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
 
     def put_character(
         self,
@@ -444,6 +458,8 @@ class Mai2ItemData(BaseData):
         card_kind: int,
         chara_id: int,
         map_id: int,
+        start_date: datetime,
+        end_date: datetime,
     ) -> Optional[Row]:
         sql = insert(card).values(
             user=user_id,
@@ -451,9 +467,13 @@ class Mai2ItemData(BaseData):
             cardTypeId=card_kind,
             charaId=chara_id,
             mapId=map_id,
+            startDate=start_date,
+            endDate=end_date,
         )
 
-        conflict = sql.on_duplicate_key_update(charaId=chara_id, mapId=map_id)
+        conflict = sql.on_duplicate_key_update(
+            charaId=chara_id, mapId=map_id, startDate=start_date, endDate=end_date
+        )
 
         result = self.execute(conflict)
         if result is None:

titles/mai2/schema/profile.py

@@ -99,6 +99,68 @@ detail = Table(
     mysql_charset="utf8mb4",
 )
 
+detail_old = Table(
+   "maimai_profile_detail",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("lastDataVersion", Integer),
+    Column("userName", String(8)),
+    Column("point", Integer),
+    Column("totalPoint", Integer),
+    Column("iconId", Integer),
+    Column("nameplateId", Integer),
+    Column("frameId", Integer),
+    Column("trophyId", Integer),
+    Column("playCount", Integer),
+    Column("playVsCount", Integer),
+    Column("playSyncCount", Integer),
+    Column("winCount", Integer),
+    Column("helpCount", Integer),
+    Column("comboCount", Integer),
+    Column("feverCount", Integer),
+    Column("totalHiScore", Integer),
+    Column("totalEasyHighScore", Integer),
+    Column("totalBasicHighScore", Integer),
+    Column("totalAdvancedHighScore", Integer),
+    Column("totalExpertHighScore", Integer),
+    Column("totalMasterHighScore", Integer),
+    Column("totalReMasterHighScore", Integer),
+    Column("totalHighSync", Integer),
+    Column("totalEasySync", Integer),
+    Column("totalBasicSync", Integer),
+    Column("totalAdvancedSync", Integer),
+    Column("totalExpertSync", Integer),
+    Column("totalMasterSync", Integer),
+    Column("totalReMasterSync", Integer),
+    Column("playerRating", Integer),
+    Column("highestRating", Integer),
+    Column("rankAuthTailId", Integer),
+    Column("eventWatchedDate", String(255)),
+    Column("webLimitDate", String(255)),
+    Column("challengeTrackPhase", Integer),
+    Column("firstPlayBits", Integer),
+    Column("lastPlayDate", String(255)),
+    Column("lastPlaceId", Integer),
+    Column("lastPlaceName", String(255)),
+    Column("lastRegionId", Integer),
+    Column("lastRegionName", String(255)),
+    Column("lastClientId", String(255)),
+    Column("lastCountryCode", String(255)),
+    Column("eventPoint", Integer),
+    Column("totalLv", Integer),
+    Column("lastLoginBonusDay", Integer),
+    Column("lastSurvivalBonusDay", Integer),
+    Column("loginBonusLv", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_detail_uk"),
+    mysql_charset="utf8mb4",
+)
+
 ghost = Table(
     "mai2_profile_ghost",
     metadata,
@@ -223,6 +285,99 @@ option = Table(
     mysql_charset="utf8mb4",
 )
 
+option_old = Table(
+    "maimai_profile_option",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("soudEffect", Integer),
+    Column("mirrorMode", Integer),
+    Column("guideSpeed", Integer),
+    Column("bgInfo", Integer),
+    Column("brightness", Integer),
+    Column("isStarRot", Integer),
+    Column("breakSe", Integer),
+    Column("slideSe", Integer),
+    Column("hardJudge", Integer),
+    Column("isTagJump", Integer),
+    Column("breakSeVol", Integer),
+    Column("slideSeVol", Integer),
+    Column("isUpperDisp", Integer),
+    Column("trackSkip", Integer),
+    Column("optionMode", Integer),
+    Column("simpleOptionParam", Integer),
+    Column("adjustTiming", Integer),
+    Column("dispTiming", Integer),
+    Column("timingPos", Integer),
+    Column("ansVol", Integer),
+    Column("noteVol", Integer),
+    Column("dmgVol", Integer),
+    Column("appealFlame", Integer),
+    Column("isFeverDisp", Integer),
+    Column("dispJudge", Integer),
+    Column("judgePos", Integer),
+    Column("ratingGuard", Integer),
+    Column("selectChara", Integer),
+    Column("sortType", Integer),
+    Column("filterGenre", Integer),
+    Column("filterLevel", Integer),
+    Column("filterRank", Integer),
+    Column("filterVersion", Integer),
+    Column("filterRec", Integer),
+    Column("filterFullCombo", Integer),
+    Column("filterAllPerfect", Integer),
+    Column("filterDifficulty", Integer),
+    Column("filterFullSync", Integer),
+    Column("filterReMaster", Integer),
+    Column("filterMaxFever", Integer),
+    Column("finalSelectId", Integer),
+    Column("finalSelectCategory", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_option_uk"),
+    mysql_charset="utf8mb4",
+)
+
+web_opt = Table(
+    "maimai_profile_web_option",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("isNetMember", Boolean),
+    Column("dispRate", Integer),
+    Column("dispJudgeStyle", Integer),
+    Column("dispRank", Integer),
+    Column("dispHomeRanker", Integer),
+    Column("dispTotalLv", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_web_option_uk"),
+    mysql_charset="utf8mb4",
+)
+
+grade_status = Table(
+    "maimai_profile_grade_status",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("gradeVersion", Integer),
+    Column("gradeLevel", Integer),
+    Column("gradeSubLevel", Integer),
+    Column("gradeMaxId", Integer),
+    UniqueConstraint("user", "gradeVersion", name="maimai_profile_grade_status_uk"),
+    mysql_charset="utf8mb4",
+)
+
 rating = Table(
     "mai2_profile_rating",
     metadata,
@@ -268,42 +423,91 @@ activity = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("kind", Integer, nullable=False),
-    Column("activityId", Integer, nullable=False),
-    Column("param1", Integer, nullable=False),
-    Column("param2", Integer, nullable=False),
-    Column("param3", Integer, nullable=False),
-    Column("param4", Integer, nullable=False),
-    Column("sortNumber", Integer, nullable=False),
+    Column("kind", Integer),
+    Column("activityId", Integer),
+    Column("param1", Integer),
+    Column("param2", Integer),
+    Column("param3", Integer),
+    Column("param4", Integer),
+    Column("sortNumber", Integer),
     UniqueConstraint("user", "kind", "activityId", name="mai2_profile_activity_uk"),
     mysql_charset="utf8mb4",
 )
 
+boss = Table(
+    "maimai_profile_boss",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("pandoraFlagList0", Integer),
+    Column("pandoraFlagList1", Integer),
+    Column("pandoraFlagList2", Integer),
+    Column("pandoraFlagList3", Integer),
+    Column("pandoraFlagList4", Integer),
+    Column("pandoraFlagList5", Integer),
+    Column("pandoraFlagList6", Integer),
+    Column("emblemFlagList", Integer),
+    UniqueConstraint("user", name="mai2_profile_boss_uk"),
+    mysql_charset="utf8mb4",
+)
+
+recent_rating = Table(
+    "maimai_profile_recent_rating",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("userRecentRatingList", JSON),
+    UniqueConstraint("user", name="mai2_profile_recent_rating_uk"),
+    mysql_charset="utf8mb4",
+)
+
+consec_logins = Table(
+        "mai2_profile_consec_logins",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("version", Integer, nullable=False),
+    Column("logins", Integer),
+    UniqueConstraint("user", "version", name="mai2_profile_consec_logins_uk"),
+    mysql_charset="utf8mb4",
+)
 
 class Mai2ProfileData(BaseData):
     def put_profile_detail(
-        self, user_id: int, version: int, detail_data: Dict
+        self, user_id: int, version: int, detail_data: Dict, is_dx: bool = True
     ) -> Optional[Row]:
         detail_data["user"] = user_id
         detail_data["version"] = version
-        sql = insert(detail).values(**detail_data)
+
+        if is_dx:
+            sql = insert(detail).values(**detail_data)
+        else:
+            sql = insert(detail_old).values(**detail_data)
 
         conflict = sql.on_duplicate_key_update(**detail_data)
 
         result = self.execute(conflict)
         if result is None:
             self.logger.warn(
-                f"put_profile: Failed to create profile! user_id {user_id}"
+                f"put_profile: Failed to create profile! user_id {user_id} is_dx {is_dx}"
             )
             return None
         return result.lastrowid
 
-    def get_profile_detail(self, user_id: int, version: int) -> Optional[Row]:
-        sql = (
-            select(detail)
-            .where(and_(detail.c.user == user_id, detail.c.version <= version))
-            .order_by(detail.c.version.desc())
-        )
+    def get_profile_detail(self, user_id: int, version: int, is_dx: bool = True) -> Optional[Row]:
+        if is_dx:
+            sql = (
+                select(detail)
+                .where(and_(detail.c.user == user_id, detail.c.version <= version))
+                .order_by(detail.c.version.desc())
+            )
+        
+        else:
+            sql = (
+                select(detail_old)
+                .where(and_(detail_old.c.user == user_id, detail_old.c.version <= version))
+                .order_by(detail_old.c.version.desc())
+            )
 
         result = self.execute(sql)
         if result is None:
@@ -365,26 +569,36 @@ class Mai2ProfileData(BaseData):
         return result.fetchone()
 
     def put_profile_option(
-        self, user_id: int, version: int, option_data: Dict
+        self, user_id: int, version: int, option_data: Dict, is_dx: bool = True
     ) -> Optional[int]:
         option_data["user"] = user_id
         option_data["version"] = version
 
-        sql = insert(option).values(**option_data)
+        if is_dx:
+            sql = insert(option).values(**option_data)
+        else:
+            sql = insert(option_old).values(**option_data)
         conflict = sql.on_duplicate_key_update(**option_data)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"put_profile_option: failed to update! {user_id}")
+            self.logger.warn(f"put_profile_option: failed to update! {user_id} is_dx {is_dx}")
             return None
         return result.lastrowid
 
-    def get_profile_option(self, user_id: int, version: int) -> Optional[Row]:
-        sql = (
-            select(option)
-            .where(and_(option.c.user == user_id, option.c.version <= version))
-            .order_by(option.c.version.desc())
-        )
+    def get_profile_option(self, user_id: int, version: int, is_dx: bool = True) -> Optional[Row]:
+        if is_dx:
+            sql = (
+                select(option)
+                .where(and_(option.c.user == user_id, option.c.version <= version))
+                .order_by(option.c.version.desc())
+            )
+        else:
+            sql = (
+                select(option_old)
+                .where(and_(option_old.c.user == user_id, option_old.c.version <= version))
+                .order_by(option_old.c.version.desc())
+            )
 
         result = self.execute(sql)
         if result is None:
@@ -474,3 +688,130 @@ class Mai2ProfileData(BaseData):
         if result is None:
             return None
         return result.fetchall()
+
+    def put_web_option(self, user_id: int, version: int, web_opts: Dict) -> Optional[int]:
+        web_opts["user"] = user_id
+        web_opts["version"] = version
+        sql = insert(web_opt).values(**web_opts)
+
+        conflict = sql.on_duplicate_key_update(**web_opts)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_web_option: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_web_option(self, user_id: int, version: int) -> Optional[Row]:
+        sql = web_opt.select(and_(web_opt.c.user == user_id, web_opt.c.version == version))
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_grade_status(self, user_id: int, grade_stat: Dict) -> Optional[int]:
+        grade_stat["user"] = user_id
+        sql = insert(grade_status).values(**grade_stat)
+
+        conflict = sql.on_duplicate_key_update(**grade_stat)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_grade_status: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_grade_status(self, user_id: int) -> Optional[Row]:
+        sql = grade_status.select(grade_status.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_boss_list(self, user_id: int, boss_stat: Dict) -> Optional[int]:
+        boss_stat["user"] = user_id
+        sql = insert(boss).values(**boss_stat)
+
+        conflict = sql.on_duplicate_key_update(**boss_stat)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_boss_list: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_boss_list(self, user_id: int) -> Optional[Row]:
+        sql = boss.select(boss.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_recent_rating(self, user_id: int, rr: Dict) -> Optional[int]:
+        sql = insert(recent_rating).values(user=user_id, userRecentRatingList=rr)
+
+        conflict = sql.on_duplicate_key_update({'userRecentRatingList': rr})
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_recent_rating: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_recent_rating(self, user_id: int) -> Optional[Row]:
+        sql = recent_rating.select(recent_rating.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def add_consec_login(self, user_id: int, version: int) -> None:
+        sql = insert(consec_logins).values(
+            user=user_id,
+            version=version,
+            logins=1
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            logins=consec_logins.c.logins + 1
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to update consecutive login count for user {user_id} version {version}")
+
+    def get_consec_login(self, user_id: int, version: int) -> Optional[Row]:
+        sql = select(consec_logins).where(and_(
+            consec_logins.c.user==user_id,
+            consec_logins.c.version==version,
+        ))
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def reset_consec_login(self, user_id: int, version: int) -> Optional[Row]:
+        sql = consec_logins.update(and_(
+            consec_logins.c.user==user_id,
+            consec_logins.c.version==version,
+        )).values(
+            logins=1
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()

titles/mai2/schema/score.py

@@ -7,6 +7,7 @@ from sqlalchemy.engine import Row
 from sqlalchemy.dialects.mysql import insert
 
 from core.data.schema import BaseData, metadata
+from core.data import cached
 
 best_score = Table(
     "mai2_score_best",
@@ -174,29 +175,137 @@ course = Table(
     mysql_charset="utf8mb4",
 )
 
+playlog_old = Table(
+    "maimai_playlog",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer),
+    # Pop access code
+    Column("orderId", Integer),
+    Column("sortNumber", Integer),
+    Column("placeId", Integer),
+    Column("placeName", String(255)),
+    Column("country", String(255)),
+    Column("regionId", Integer),
+    Column("playDate", String(255)),
+    Column("userPlayDate", String(255)),
+    Column("musicId", Integer),
+    Column("level", Integer),
+    Column("gameMode", Integer),
+    Column("rivalNum", Integer),
+    Column("track", Integer),
+    Column("eventId", Integer),
+    Column("isFreeToPlay", Boolean),
+    Column("playerRating", Integer),
+    Column("playedUserId1", Integer),
+    Column("playedUserId2", Integer),
+    Column("playedUserId3", Integer),
+    Column("playedUserName1", String(255)),
+    Column("playedUserName2", String(255)),
+    Column("playedUserName3", String(255)),
+    Column("playedMusicLevel1", Integer),
+    Column("playedMusicLevel2", Integer),
+    Column("playedMusicLevel3", Integer),
+    Column("achievement", Integer),
+    Column("score", Integer),
+    Column("tapScore", Integer),
+    Column("holdScore", Integer),
+    Column("slideScore", Integer),
+    Column("breakScore", Integer),
+    Column("syncRate", Integer),
+    Column("vsWin", Integer),
+    Column("isAllPerfect", Boolean),
+    Column("fullCombo", Integer),
+    Column("maxFever", Integer),
+    Column("maxCombo", Integer),
+    Column("tapPerfect", Integer),
+    Column("tapGreat", Integer),
+    Column("tapGood", Integer),
+    Column("tapBad", Integer),
+    Column("holdPerfect", Integer),
+    Column("holdGreat", Integer),
+    Column("holdGood", Integer),
+    Column("holdBad", Integer),
+    Column("slidePerfect", Integer),
+    Column("slideGreat", Integer),
+    Column("slideGood", Integer),
+    Column("slideBad", Integer),
+    Column("breakPerfect", Integer),
+    Column("breakGreat", Integer),
+    Column("breakGood", Integer),
+    Column("breakBad", Integer),
+    Column("judgeStyle", Integer),
+    Column("isTrackSkip", Boolean),
+    Column("isHighScore", Boolean),
+    Column("isChallengeTrack", Boolean),
+    Column("challengeLife", Integer),
+    Column("challengeRemain", Integer),
+    Column("isAllPerfectPlus", Integer),
+    mysql_charset="utf8mb4",
+)
+
+best_score_old = Table(
+    "maimai_score_best",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("musicId", Integer),
+    Column("level", Integer),
+    Column("playCount", Integer),
+    Column("achievement", Integer),
+    Column("scoreMax", Integer),
+    Column("syncRateMax", Integer),
+    Column("isAllPerfect", Boolean),
+    Column("isAllPerfectPlus", Integer),
+    Column("fullCombo", Integer),
+    Column("maxFever", Integer),
+    UniqueConstraint("user", "musicId", "level", name="maimai_score_best_uk"),
+    mysql_charset="utf8mb4",
+)
 
 class Mai2ScoreData(BaseData):
-    def put_best_score(self, user_id: int, score_data: Dict) -> Optional[int]:
+    def put_best_score(self, user_id: int, score_data: Dict, is_dx: bool = True) -> Optional[int]:
         score_data["user"] = user_id
-        sql = insert(best_score).values(**score_data)
 
+        if is_dx:
+            sql = insert(best_score).values(**score_data)
+        else:
+            sql = insert(best_score_old).values(**score_data)
         conflict = sql.on_duplicate_key_update(**score_data)
 
         result = self.execute(conflict)
         if result is None:
             self.logger.error(
-                f"put_best_score:  Failed to insert best score! user_id {user_id}"
+                f"put_best_score:  Failed to insert best score! user_id {user_id} is_dx {is_dx}"
             )
             return None
         return result.lastrowid
 
-    def get_best_scores(self, user_id: int, song_id: int = None) -> Optional[List[Row]]:
-        sql = best_score.select(
-            and_(
-                best_score.c.user == user_id,
-                (best_score.c.song_id == song_id) if song_id is not None else True,
+    @cached(2)
+    def get_best_scores(self, user_id: int, song_id: int = None, is_dx: bool = True) -> Optional[List[Row]]:
+        if is_dx:
+            sql = best_score.select(
+                and_(
+                    best_score.c.user == user_id,
+                    (best_score.c.song_id == song_id) if song_id is not None else True,
+                )
+            )
+        else:
+            sql = best_score_old.select(
+                and_(
+                    best_score_old.c.user == user_id,
+                    (best_score_old.c.song_id == song_id) if song_id is not None else True,
+                )
             )
-        )
 
         result = self.execute(sql)
         if result is None:
@@ -219,15 +328,19 @@ class Mai2ScoreData(BaseData):
             return None
         return result.fetchone()
 
-    def put_playlog(self, user_id: int, playlog_data: Dict) -> Optional[int]:
+    def put_playlog(self, user_id: int, playlog_data: Dict, is_dx: bool = True) -> Optional[int]:
         playlog_data["user"] = user_id
-        sql = insert(playlog).values(**playlog_data)
+
+        if is_dx:
+            sql = insert(playlog).values(**playlog_data)
+        else:
+            sql = insert(playlog_old).values(**playlog_data)
 
         conflict = sql.on_duplicate_key_update(**playlog_data)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.error(f"put_playlog:  Failed to insert! user_id {user_id}")
+            self.logger.error(f"put_playlog:  Failed to insert! user_id {user_id} is_dx {is_dx}")
             return None
         return result.lastrowid
 
@@ -249,4 +362,4 @@ class Mai2ScoreData(BaseData):
         result = self.execute(sql)
         if result is None:
             return None
-        return result.fetchone()
+        return result.fetchall()

titles/mai2/splash.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2Splash(Mai2Base):
+class Mai2Splash(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_SPLASH

titles/mai2/splashplus.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2SplashPlus(Mai2Base):
+class Mai2SplashPlus(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS

titles/mai2/universe.py

@@ -5,12 +5,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.const import Mai2Constants
 from titles.mai2.config import Mai2Config
 
 
-class Mai2Universe(Mai2Base):
+class Mai2Universe(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
@@ -104,8 +104,12 @@ class Mai2Universe(Mai2Base):
             tmp.pop("id")
             tmp.pop("user")
 
-            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
-            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["startDate"] = datetime.strftime(
+                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["endDate"] = datetime.strftime(
+                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
             card_list.append(tmp)
 
         return {
@@ -154,6 +158,10 @@ class Mai2Universe(Mai2Base):
         # set a random card serial number
         serial_id = "".join([str(randint(0, 9)) for _ in range(20)])
 
+        # calculate start and end date of the card
+        start_date = datetime.utcnow()
+        end_date = datetime.utcnow() + timedelta(days=15)
+
         user_card = upsert["userCard"]
         self.data.item.put_card(
             user_id,
@@ -161,8 +169,26 @@ class Mai2Universe(Mai2Base):
             user_card["cardTypeId"],
             user_card["charaId"],
             user_card["mapId"],
+            # add the correct start date and also the end date in 15 days
+            start_date,
+            end_date,
         )
 
+        # get the profile extend to save the new bought card
+        extend = self.data.profile.get_profile_extend(user_id, self.version)
+        if extend:
+            extend = extend._asdict()
+            # parse the selectedCardList
+            # 6 = Freedom Pass, 4 = Gold Pass (cardTypeId)
+            selected_cards: List = extend["selectedCardList"]
+
+            # if no pass is already added, add the corresponding pass
+            if not user_card["cardTypeId"] in selected_cards:
+                selected_cards.insert(0, user_card["cardTypeId"])
+
+            extend["selectedCardList"] = selected_cards
+            self.data.profile.put_profile_extend(user_id, self.version, extend)
+
         # properly format userPrintDetail for the database
         upsert.pop("userCard")
         upsert.pop("serialId")
@@ -174,8 +200,8 @@ class Mai2Universe(Mai2Base):
             "returnCode": 1,
             "orderId": 0,
             "serialId": serial_id,
-            "startDate": "2018-01-01 00:00:00",
-            "endDate": "2038-01-01 00:00:00",
+            "startDate": datetime.strftime(start_date, Mai2Constants.DATE_TIME_FORMAT),
+            "endDate": datetime.strftime(end_date, Mai2Constants.DATE_TIME_FORMAT),
         }
 
     def handle_cm_upsert_user_printlog_api_request(self, data: Dict) -> Dict:

titles/mai2/universeplus.py

@@ -1,12 +1,12 @@
 from typing import Dict
 
 from core.config import CoreConfig
-from titles.mai2.universe import Mai2Universe
+from titles.mai2.dx import Mai2DX
 from titles.mai2.const import Mai2Constants
 from titles.mai2.config import Mai2Config
 
 
-class Mai2UniversePlus(Mai2Universe):
+class Mai2UniversePlus(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS

titles/ongeki/base.py

@@ -268,7 +268,7 @@ class OngekiBase:
         }
 
     def handle_get_game_id_list_api_request(self, data: Dict) -> Dict:
-        game_idlist: list[str, Any] = []  # 1 to 230 & 8000 to 8050
+        game_idlist: List[str, Any] = []  # 1 to 230 & 8000 to 8050
 
         if data["type"] == 1:
             for i in range(1, 231):
@@ -443,7 +443,7 @@ class OngekiBase:
                 "userItemList": [],
             }
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(data["nextIndex"] % 10000000000, len(p)):
             if len(items) > data["maxCount"]:
                 break

titles/pokken/base.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
 import json, logging
-from typing import Any, Dict
+from typing import Any, Dict, List
 import random
 
 from core.data import Data
@@ -8,6 +8,7 @@ from core import CoreConfig
 from .config import PokkenConfig
 from .proto import jackal_pb2
 from .database import PokkenData
+from .const import PokkenConstants
 
 
 class PokkenBase:
@@ -44,19 +45,19 @@ class PokkenBase:
         biwa_setting = {
             "MatchingServer": {
                 "host": f"https://{self.game_cfg.server.hostname}",
-                "port": self.game_cfg.server.port,
+                "port": self.game_cfg.ports.game,
                 "url": "/SDAK/100/matching",
             },
             "StunServer": {
-                "addr": self.game_cfg.server.hostname,
-                "port": self.game_cfg.server.port_stun,
+                "addr": self.game_cfg.server.stun_server_host,
+                "port": self.game_cfg.server.stun_server_port,
             },
             "TurnServer": {
-                "addr": self.game_cfg.server.hostname,
-                "port": self.game_cfg.server.port_turn,
+                "addr": self.game_cfg.server.stun_server_host,
+                "port": self.game_cfg.server.stun_server_port,
             },
-            "AdmissionUrl": f"ws://{self.game_cfg.server.hostname}:{self.game_cfg.server.port_admission}",
-            "locationId": 123,
+            "AdmissionUrl": f"ws://{self.game_cfg.server.hostname}:{self.game_cfg.ports.admission}",
+            "locationId": 123, # FIXME: Get arcade's ID from the database
             "logfilename": "JackalMatchingLibrary.log",
             "biwalogfilename": "./biwa.log",
         }
@@ -94,6 +95,7 @@ class PokkenBase:
         res.type = jackal_pb2.MessageType.LOAD_CLIENT_SETTINGS
         settings = jackal_pb2.LoadClientSettingsResponseData()
 
+        # TODO: Make configurable
         settings.money_magnification = 1
         settings.continue_bonus_exp = 100
         settings.continue_fight_money = 100
@@ -274,6 +276,100 @@ class PokkenBase:
         res.result = 1
         res.type = jackal_pb2.MessageType.SAVE_USER
 
+        req = request.save_user
+        user_id = req.banapass_id
+        
+        tut_flgs: List[int] = []
+        ach_flgs: List[int] = []
+        evt_flgs: List[int] = []
+        evt_params: List[int] = []
+
+        get_rank_pts: int = req.get_trainer_rank_point if req.get_trainer_rank_point else 0
+        get_money: int = req.get_money
+        get_score_pts: int = req.get_score_point if req.get_score_point else 0
+        grade_max: int = req.grade_max_num
+        extra_counter: int = req.extra_counter
+        evt_reward_get_flg: int = req.event_reward_get_flag
+        num_continues: int = req.continue_num
+        total_play_days: int = req.total_play_days
+        awake_num: int = req.awake_num  # ?
+        use_support_ct: int = req.use_support_num
+        beat_num: int = req.beat_num # ?
+        evt_state: int = req.event_state
+        aid_skill: int = req.aid_skill
+        last_evt: int = req.last_play_event_id
+
+        battle = req.battle_data
+        mon = req.pokemon_data
+
+        p = self.data.profile.touch_profile(user_id)
+        if p is None or not p:
+            self.data.profile.create_profile(user_id)
+
+        if req.trainer_name_pending is not None and req.trainer_name_pending: # we're saving for the first time
+            self.data.profile.set_profile_name(user_id, req.trainer_name_pending, req.avatar_gender if req.avatar_gender else None)
+
+        for tut_flg in req.tutorial_progress_flag:
+            tut_flgs.append(tut_flg)
+        
+        self.data.profile.update_profile_tutorial_flags(user_id, tut_flgs)
+
+        for ach_flg in req.achievement_flag:
+            ach_flgs.append(ach_flg)
+        
+        self.data.profile.update_profile_tutorial_flags(user_id, ach_flg)
+
+        for evt_flg in req.event_achievement_flag:
+            evt_flgs.append(evt_flg)
+
+        for evt_param in req.event_achievement_param:
+            evt_params.append(evt_param)
+
+        self.data.profile.update_profile_event(user_id, evt_state, evt_flgs, evt_params, )
+        
+        for reward in req.reward_data:
+            self.data.item.add_reward(user_id, reward.get_category_id, reward.get_content_id, reward.get_type_id)
+        
+        self.data.profile.add_profile_points(user_id, get_rank_pts, get_money, get_score_pts, grade_max)
+        
+        self.data.profile.update_support_team(user_id, 1, req.support_set_1[0], req.support_set_1[1])
+        self.data.profile.update_support_team(user_id, 2, req.support_set_2[0], req.support_set_2[1])
+        self.data.profile.update_support_team(user_id, 3, req.support_set_3[0], req.support_set_3[1])
+
+        self.data.profile.put_pokemon(user_id, mon.char_id, mon.illustration_book_no, mon.bp_point_atk, mon.bp_point_res, mon.bp_point_def, mon.bp_point_sp)
+        self.data.profile.add_pokemon_xp(user_id, mon.char_id, mon.get_pokemon_exp)
+        
+        for x in range(len(battle.play_mode)):
+            self.data.profile.put_pokemon_battle_result(
+                user_id, 
+                mon.char_id, 
+                PokkenConstants.BATTLE_TYPE(battle.play_mode[x]), 
+                PokkenConstants.BATTLE_RESULT(battle.result[x])
+            )
+
+        self.data.profile.put_stats(
+            user_id,
+            battle.ex_ko_num,
+            battle.wko_num,
+            battle.timeup_win_num,
+            battle.cool_ko_num,
+            battle.perfect_ko_num,
+            num_continues
+        )
+
+        self.data.profile.put_extra(
+            user_id,
+            extra_counter,
+            evt_reward_get_flg,
+            total_play_days,
+            awake_num,
+            use_support_ct,
+            beat_num,
+            aid_skill,
+            last_evt
+        )
+
+
         return res.SerializeToString()
 
     def handle_save_ingame_log(self, data: jackal_pb2.Request) -> bytes:
@@ -302,16 +398,35 @@ class PokkenBase:
         self, data: Dict = {}, client_ip: str = "127.0.0.1"
     ) -> Dict:
         """
-        "sessionId":"12345678",
+                "sessionId":"12345678",
+                "A":{
+                    "pcb_id": data["data"]["must"]["pcb_id"],
+                    "gip": client_ip
+                },
+        """
+        return {
+            "data": {
+                "sessionId":"12345678",
                 "A":{
                     "pcb_id": data["data"]["must"]["pcb_id"],
                     "gip": client_ip
                 },
                 "list":[]
-        """
-        return {}
+            }
+        }
 
     def handle_matching_stop_matching(
         self, data: Dict = {}, client_ip: str = "127.0.0.1"
     ) -> Dict:
         return {}
+
+    def handle_admission_noop(self, data: Dict, req_ip: str = "127.0.0.1") -> Dict:
+        return {}
+    
+    def handle_admission_joinsession(self, data: Dict, req_ip: str = "127.0.0.1") -> Dict:
+        self.logger.info(f"Admission: JoinSession from {req_ip}")
+        return {
+            'data': {
+                "id": 12345678
+            }
+        }

titles/pokken/config.py

@@ -25,30 +25,6 @@ class PokkenServerConfig:
             )
         )
 
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port", default=9000
-        )
-
-    @property
-    def port_stun(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port_stun", default=9001
-        )
-
-    @property
-    def port_turn(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port_turn", default=9002
-        )
-
-    @property
-    def port_admission(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port_admission", default=9003
-        )
-
     @property
     def auto_register(self) -> bool:
         """
@@ -59,7 +35,51 @@ class PokkenServerConfig:
             self.__config, "pokken", "server", "auto_register", default=True
         )
 
+    @property
+    def enable_matching(self) -> bool:
+        """
+        If global matching should happen
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "server", "enable_matching", default=False
+        )
+    
+    @property
+    def stun_server_host(self) -> str:
+        """
+        Hostname of the EXTERNAL stun server the game should connect to. This is not handled by artemis.
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "server", "stun_server_host", default="stunserver.stunprotocol.org"
+        )
+
+    @property
+    def stun_server_port(self) -> int:
+        """
+        Port of the EXTERNAL stun server the game should connect to. This is not handled by artemis.
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "server", "stun_server_port", default=3478
+        )
+
+class PokkenPortsConfig:
+    def __init__(self, parent_config: "PokkenConfig"):
+        self.__config = parent_config
+    
+    @property
+    def game(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "ports", "game", default=9000
+        )
+
+    @property
+    def admission(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "ports", "admission", default=9001
+        )
+    
 
 class PokkenConfig(dict):
     def __init__(self) -> None:
         self.server = PokkenServerConfig(self)
+        self.ports = PokkenPortsConfig(self)

titles/pokken/const.py

@@ -11,14 +11,14 @@ class PokkenConstants:
     VERSION_NAMES = "Pokken Tournament"
 
     class BATTLE_TYPE(Enum):
-        BATTLE_TYPE_TUTORIAL = 1
-        BATTLE_TYPE_AI = 2
-        BATTLE_TYPE_LAN = 3
-        BATTLE_TYPE_WAN = 4
+        TUTORIAL = 1
+        AI = 2
+        LAN = 3
+        WAN = 4
 
     class BATTLE_RESULT(Enum):
-        BATTLE_RESULT_WIN = 1
-        BATTLE_RESULT_LOSS = 2
+        WIN = 1
+        LOSS = 2
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/pokken/index.py

@@ -1,6 +1,7 @@
 from typing import Tuple
 from twisted.web.http import Request
 from twisted.web import resource
+from twisted.internet import reactor
 import json, ast
 from datetime import datetime
 import yaml
@@ -11,10 +12,11 @@ from os import path
 from google.protobuf.message import DecodeError
 
 from core import CoreConfig, Utils
-from titles.pokken.config import PokkenConfig
-from titles.pokken.base import PokkenBase
-from titles.pokken.const import PokkenConstants
-from titles.pokken.proto import jackal_pb2
+from .config import PokkenConfig
+from .base import PokkenBase
+from .const import PokkenConstants
+from .proto import jackal_pb2
+from .services import PokkenAdmissionFactory
 
 
 class PokkenServlet(resource.Resource):
@@ -69,7 +71,7 @@ class PokkenServlet(resource.Resource):
 
         return (
             True,
-            f"https://{game_cfg.server.hostname}:{game_cfg.server.port}/{game_code}/$v/",
+            f"https://{game_cfg.server.hostname}:{game_cfg.ports.game}/{game_code}/$v/",
             f"{game_cfg.server.hostname}/SDAK/$v/",
         )
 
@@ -90,8 +92,10 @@ class PokkenServlet(resource.Resource):
         return (True, "PKF1")
 
     def setup(self) -> None:
-        # TODO: Setup stun, turn (UDP) and admission (WSS) servers
-        pass
+        if self.game_cfg.server.enable_matching:
+            reactor.listenTCP(
+                self.game_cfg.ports.admission, PokkenAdmissionFactory(self.core_cfg, self.game_cfg)
+            )
 
     def render_POST(
         self, request: Request, version: int = 0, endpoints: str = ""
@@ -128,6 +132,9 @@ class PokkenServlet(resource.Resource):
         return ret
 
     def handle_matching(self, request: Request) -> bytes:
+        if not self.game_cfg.server.enable_matching:
+            return b""
+        
         content = request.content.getvalue()
         client_ip = Utils.get_ip_addr(request)
 

titles/pokken/schema/item.py

@@ -31,4 +31,16 @@ class PokkenItemData(BaseData):
     Items obtained as rewards
     """
 
-    pass
+    def add_reward(self, user_id: int, category: int, content: int, item_type: int) -> Optional[int]:
+        sql = insert(item).values(
+            user=user_id,
+            category=category,
+            content=content,
+            type=item_type,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to insert reward for user {user_id}: {category}-{content}-{item_type}")
+            return None
+        return result.lastrowid

titles/pokken/schema/profile.py

@@ -1,4 +1,4 @@
-from typing import Optional, Dict, List
+from typing import Optional, Dict, List, Union
 from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, case
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
 from sqlalchemy.schema import ForeignKey
@@ -125,7 +125,7 @@ pokemon_data = Table(
     Column("win_vs_lan", Integer),
     Column("battle_num_vs_cpu", Integer),  # 2
     Column("win_cpu", Integer),
-    Column("battle_all_num_tutorial", Integer),
+    Column("battle_all_num_tutorial", Integer), # ???
     Column("battle_num_tutorial", Integer),  # 1?
     Column("bp_point_atk", Integer),
     Column("bp_point_res", Integer),
@@ -137,6 +137,14 @@ pokemon_data = Table(
 
 
 class PokkenProfileData(BaseData):
+    def touch_profile(self, user_id: int) -> Optional[int]:
+        sql = select([profile.c.id]).where(profile.c.user == user_id)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()['id']
+
     def create_profile(self, user_id: int) -> Optional[int]:
         sql = insert(profile).values(user=user_id)
         conflict = sql.on_duplicate_key_update(user=user_id)
@@ -147,11 +155,10 @@ class PokkenProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def set_profile_name(self, user_id: int, new_name: str) -> None:
-        sql = (
-            update(profile)
-            .where(profile.c.user == user_id)
-            .values(trainer_name=new_name)
+    def set_profile_name(self, user_id: int, new_name: str, gender: Union[int, None] = None) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            trainer_name=new_name,
+            avatar_gender=gender if gender is not None else profile.c.avatar_gender
         )
         result = self.execute(sql)
         if result is None:
@@ -159,13 +166,75 @@ class PokkenProfileData(BaseData):
                 f"Failed to update pokken profile name for user {user_id}!"
             )
 
-    def update_profile_tutorial_flags(self, user_id: int, tutorial_flags: Dict) -> None:
-        pass
+    def put_extra(
+        self, 
+        user_id: int, 
+        extra_counter: int,
+        evt_reward_get_flg: int,
+        total_play_days: int,
+        awake_num: int,
+        use_support_ct: int,
+        beat_num: int,
+        aid_skill: int,
+        last_evt: int
+    ) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            extra_counter=extra_counter,
+            event_reward_get_flag=evt_reward_get_flg,
+            total_play_days=total_play_days,
+            awake_num=awake_num,
+            use_support_num=use_support_ct,
+            beat_num=beat_num,
+            aid_skill=aid_skill,
+            last_play_event_id=last_evt
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to put extra data for user {user_id}")
+
+    def update_profile_tutorial_flags(self, user_id: int, tutorial_flags: List) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            tutorial_progress_flag=tutorial_flags,
+        )
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to update pokken profile tutorial flags for user {user_id}!"
+            )
+
+    def update_profile_achievement_flags(self, user_id: int, achievement_flags: List) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            achievement_flag=achievement_flags,
+        )
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to update pokken profile achievement flags for user {user_id}!"
+            )
+
+    def update_profile_event(self, user_id: int, event_state: List, event_flags: List[int], event_param: List[int], last_evt: int = None) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            event_state=event_state,
+            event_achievement_flag=event_flags,
+            event_achievement_param=event_param,
+            last_play_event_id=last_evt if last_evt is not None else profile.c.last_play_event_id,
+        )
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to update pokken profile event state for user {user_id}!"
+            )
 
     def add_profile_points(
-        self, user_id: int, rank_pts: int, money: int, score_pts: int
+        self, user_id: int, rank_pts: int, money: int, score_pts: int, grade_max: int
     ) -> None:
-        pass
+        sql = update(profile).where(profile.c.user == user_id).values(
+            trainer_rank_point = profile.c.trainer_rank_point + rank_pts,
+            fight_money = profile.c.fight_money + money,
+            score_point = profile.c.score_point + score_pts,
+            grade_max_num = grade_max
+        )
 
     def get_profile(self, user_id: int) -> Optional[Row]:
         sql = profile.select(profile.c.user == user_id)
@@ -174,18 +243,53 @@ class PokkenProfileData(BaseData):
             return None
         return result.fetchone()
 
-    def put_pokemon_data(
+    def put_pokemon(
         self,
         user_id: int,
         pokemon_id: int,
         illust_no: int,
-        get_exp: int,
         atk: int,
         res: int,
         defe: int,
-        sp: int,
+        sp: int
     ) -> Optional[int]:
-        pass
+        sql = insert(pokemon_data).values(
+            user=user_id,
+            char_id=pokemon_id,
+            illustration_book_no=illust_no,
+            bp_point_atk=atk,
+            bp_point_res=res,
+            bp_point_defe=defe,
+            bp_point_sp=sp,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            illustration_book_no=illust_no,
+            bp_point_atk=atk,
+            bp_point_res=res,
+            bp_point_defe=defe,
+            bp_point_sp=sp,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(f"Failed to insert pokemon ID {pokemon_id} for user {user_id}")
+            return None
+        return result.lastrowid
+
+    def add_pokemon_xp(
+        self,
+        user_id: int,
+        pokemon_id: int,
+        xp: int
+    ) -> None:
+        sql = update(pokemon_data).where(and_(pokemon_data.c.user==user_id, pokemon_data.c.char_id==pokemon_id)).values(
+            pokemon_exp=pokemon_data.c.pokemon_exp + xp
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to add {xp} XP to pokemon ID {pokemon_id} for user {user_id}")
 
     def get_pokemon_data(self, user_id: int, pokemon_id: int) -> Optional[Row]:
         pass
@@ -193,13 +297,29 @@ class PokkenProfileData(BaseData):
     def get_all_pokemon_data(self, user_id: int) -> Optional[List[Row]]:
         pass
 
-    def put_results(
-        self, user_id: int, pokemon_id: int, match_type: int, match_result: int
+    def put_pokemon_battle_result(
+        self, user_id: int, pokemon_id: int, match_type: PokkenConstants.BATTLE_TYPE, match_result: PokkenConstants.BATTLE_RESULT
     ) -> None:
         """
         Records the match stats (type and win/loss) for the pokemon and profile
         """
-        pass
+        sql = update(pokemon_data).where(and_(pokemon_data.c.user==user_id, pokemon_data.c.char_id==pokemon_id)).values(
+            battle_num_tutorial=pokemon_data.c.battle_num_tutorial + 1 if match_type==PokkenConstants.BATTLE_TYPE.TUTORIAL else pokemon_data.c.battle_num_tutorial,
+            battle_all_num_tutorial=pokemon_data.c.battle_all_num_tutorial + 1 if match_type==PokkenConstants.BATTLE_TYPE.TUTORIAL else pokemon_data.c.battle_all_num_tutorial,
+
+            battle_num_vs_cpu=pokemon_data.c.battle_num_vs_cpu + 1 if match_type==PokkenConstants.BATTLE_TYPE.AI else pokemon_data.c.battle_num_vs_cpu,
+            win_cpu=pokemon_data.c.win_cpu + 1 if match_type==PokkenConstants.BATTLE_TYPE.AI and match_result==PokkenConstants.BATTLE_RESULT.WIN else pokemon_data.c.win_cpu,
+
+            battle_num_vs_lan=pokemon_data.c.battle_num_vs_lan + 1 if match_type==PokkenConstants.BATTLE_TYPE.LAN else pokemon_data.c.battle_num_vs_lan,
+            win_vs_lan=pokemon_data.c.win_vs_lan + 1 if match_type==PokkenConstants.BATTLE_TYPE.LAN and match_result==PokkenConstants.BATTLE_RESULT.WIN else pokemon_data.c.win_vs_lan,
+
+            battle_num_vs_wan=pokemon_data.c.battle_num_vs_wan + 1 if match_type==PokkenConstants.BATTLE_TYPE.WAN else pokemon_data.c.battle_num_vs_wan,
+            win_vs_wan=pokemon_data.c.win_vs_wan + 1 if match_type==PokkenConstants.BATTLE_TYPE.WAN and match_result==PokkenConstants.BATTLE_RESULT.WIN else pokemon_data.c.win_vs_wan,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to record match stats for user {user_id}'s pokemon {pokemon_id} (type {match_type.name} | result {match_result.name})")
 
     def put_stats(
         self,
@@ -214,4 +334,29 @@ class PokkenProfileData(BaseData):
         """
         Records profile stats
         """
-        pass
+        sql = update(profile).where(profile.c.user==user_id).values(
+            ex_ko_num=profile.c.ex_ko_num + exkos,
+            wko_num=profile.c.wko_num + wkos,
+            timeup_win_num=profile.c.timeup_win_num + timeout_wins,
+            cool_ko_num=profile.c.cool_ko_num + cool_kos,
+            perfect_ko_num=profile.c.perfect_ko_num + perfects,
+            continue_num=continues,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to update stats for user {user_id}")
+
+    def update_support_team(self, user_id: int, support_id: int, support1: int = 4294967295, support2: int = 4294967295) -> None:
+        sql = update(profile).where(profile.c.user==user_id).values(
+            support_set_1_1=support1 if support_id == 1 else profile.c.support_set_1_1,
+            support_set_1_2=support2 if support_id == 1 else profile.c.support_set_1_2,
+            support_set_2_1=support1 if support_id == 2 else profile.c.support_set_2_1,
+            support_set_2_2=support2 if support_id == 2 else profile.c.support_set_2_2,
+            support_set_3_1=support1 if support_id == 3 else profile.c.support_set_3_1,
+            support_set_3_2=support2 if support_id == 3 else profile.c.support_set_3_2,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to update support team {support_id} for user {user_id}")

titles/pokken/services.py

@@ -0,0 +1,66 @@
+from twisted.internet.interfaces import IAddress
+from twisted.internet.protocol import Protocol
+from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory
+from autobahn.websocket.types import ConnectionRequest
+from typing import Dict
+import logging
+import json
+
+from core.config import CoreConfig
+from .config import PokkenConfig
+from .base import PokkenBase
+
+class PokkenAdmissionProtocol(WebSocketServerProtocol):
+    def __init__(self, cfg: CoreConfig, game_cfg: PokkenConfig):
+        super().__init__()
+        self.core_config = cfg
+        self.game_config = game_cfg
+        self.logger = logging.getLogger("pokken")
+
+        self.base = PokkenBase(cfg, game_cfg)
+    
+    def onConnect(self, request: ConnectionRequest) -> None:
+        self.logger.debug(f"Admission: Connection from {request.peer}")
+    
+    def onClose(self, wasClean: bool, code: int, reason: str) -> None:
+        self.logger.debug(f"Admission: Connection with {self.transport.getPeer().host} closed {'cleanly ' if wasClean else ''}with code {code} - {reason}")
+
+    def onMessage(self, payload, isBinary: bool) -> None:
+        msg: Dict = json.loads(payload)
+        self.logger.debug(f"Admission: Message from {self.transport.getPeer().host}:{self.transport.getPeer().port} - {msg}")
+        
+        api = msg.get("api", "noop")
+        handler = getattr(self.base, f"handle_admission_{api.lower()}")
+        resp = handler(msg, self.transport.getPeer().host)
+        
+        if resp is None:
+            resp = {}
+
+        if "type" not in resp:
+            resp['type'] = "res"
+        if "data" not in resp:
+            resp['data'] = {}
+        if "api" not in resp:
+            resp['api'] = api
+        if "result" not in resp:
+            resp['result'] = 'true'
+        
+        self.logger.debug(f"Websocket response: {resp}")
+        self.sendMessage(json.dumps(resp).encode(), isBinary)
+
+class PokkenAdmissionFactory(WebSocketServerFactory):
+    protocol = PokkenAdmissionProtocol
+
+    def __init__(
+        self,
+        cfg: CoreConfig,
+        game_cfg: PokkenConfig
+    ) -> None:
+        self.core_config = cfg
+        self.game_config = game_cfg
+        super().__init__(f"ws://{self.game_config.server.hostname}:{self.game_config.ports.admission}")
+    
+    def buildProtocol(self, addr: IAddress) -> Protocol:
+        p = self.protocol(self.core_config, self.game_config)
+        p.factory = self
+        return p

titles/sao/base.py

@@ -3,7 +3,9 @@ import json, logging
 from typing import Any, Dict
 import random
 import struct
-import csv
+from csv import *
+from random import choice
+import random as rand
 
 from core.data import Data
 from core import CoreConfig
@@ -60,9 +62,26 @@ class SaoBase:
 
     def handle_c11e(self, request: Any) -> bytes:
         #common/get_auth_card_data
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "cabinet_type" / Int8ub,  # cabinet_type is a byte
+            "auth_type" / Int8ub,  # auth_type is a byte
+            "store_id_size" / Rebuild(Int32ub, len_(this.store_id) * 2),  # calculates the length of the store_id
+            "store_id" / PaddedString(this.store_id_size, "utf_16_le"),  # store_id is a (zero) padded string
+            "serial_no_size" / Rebuild(Int32ub, len_(this.serial_no) * 2),  # calculates the length of the serial_no
+            "serial_no" / PaddedString(this.serial_no_size, "utf_16_le"),  # serial_no is a (zero) padded string
+            "access_code_size" / Rebuild(Int32ub, len_(this.access_code) * 2),  # calculates the length of the access_code
+            "access_code" / PaddedString(this.access_code_size, "utf_16_le"),  # access_code is a (zero) padded string
+            "chip_id_size" / Rebuild(Int32ub, len_(this.chip_id) * 2),  # calculates the length of the chip_id
+            "chip_id" / PaddedString(this.chip_id_size, "utf_16_le"),  # chip_id is a (zero) padded string
+        )
+
+        req_data = req_struct.parse(req)
+        access_code = req_data.access_code
 
         #Check authentication
-        access_code = bytes.fromhex(request[188:268]).decode("utf-16le")
         user_id = self.core_data.card.get_user_id_from_card( access_code )
 
         if not user_id:
@@ -79,6 +98,13 @@ class SaoBase:
             self.game_data.item.put_hero_log(user_id, 102000010, 1, 0, 103000006, 0, 30086, 1001, 1002, 1003, 1005)
             self.game_data.item.put_hero_log(user_id, 103000010, 1, 0, 112000009, 0, 30086, 1001, 1002, 1003, 1005)
             self.game_data.item.put_hero_party(user_id, 0, 101000010, 102000010, 103000010)
+            self.game_data.item.put_equipment_data(user_id, 101000016, 1, 200, 0, 0, 0)
+            self.game_data.item.put_equipment_data(user_id, 103000006, 1, 200, 0, 0, 0)
+            self.game_data.item.put_equipment_data(user_id, 112000009, 1, 200, 0, 0, 0)
+            self.game_data.item.put_player_quest(user_id, 1001, True, 300, 0, 0, 1)
+
+            # Force the tutorial stage to be completed due to potential crash in-game
+            
 
         self.logger.info(f"User Authenticated: { access_code } | { user_id }")
 
@@ -87,6 +113,18 @@ class SaoBase:
 
         if user_id and not profile_data:
             profile_id = self.game_data.profile.create_profile(user_id)
+            self.game_data.item.put_hero_log(user_id, 101000010, 1, 0, 101000016, 0, 30086, 1001, 1002, 1003, 1005)
+            self.game_data.item.put_hero_log(user_id, 102000010, 1, 0, 103000006, 0, 30086, 1001, 1002, 1003, 1005)
+            self.game_data.item.put_hero_log(user_id, 103000010, 1, 0, 112000009, 0, 30086, 1001, 1002, 1003, 1005)
+            self.game_data.item.put_hero_party(user_id, 0, 101000010, 102000010, 103000010)
+            self.game_data.item.put_equipment_data(user_id, 101000016, 1, 200, 0, 0, 0)
+            self.game_data.item.put_equipment_data(user_id, 103000006, 1, 200, 0, 0, 0)
+            self.game_data.item.put_equipment_data(user_id, 112000009, 1, 200, 0, 0, 0)
+            self.game_data.item.put_player_quest(user_id, 1001, True, 300, 0, 0, 1)
+
+            # Force the tutorial stage to be completed due to potential crash in-game
+
+
             profile_data = self.game_data.profile.get_profile(user_id)
 
         resp = SaoGetAuthCardDataResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, profile_data)
@@ -99,7 +137,28 @@ class SaoBase:
 
     def handle_c104(self, request: Any) -> bytes:
         #common/login
-        access_code = bytes.fromhex(request[228:308]).decode("utf-16le")
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "cabinet_type" / Int8ub,  # cabinet_type is a byte
+            "auth_type" / Int8ub,  # auth_type is a byte
+            "store_id_size" / Rebuild(Int32ub, len_(this.store_id) * 2),  # calculates the length of the store_id
+            "store_id" / PaddedString(this.store_id_size, "utf_16_le"),  # store_id is a (zero) padded string
+            "store_name_size" / Rebuild(Int32ub, len_(this.store_name) * 2),  # calculates the length of the store_name
+            "store_name" / PaddedString(this.store_name_size, "utf_16_le"),  # store_name is a (zero) padded string
+            "serial_no_size" / Rebuild(Int32ub, len_(this.serial_no) * 2),  # calculates the length of the serial_no
+            "serial_no" / PaddedString(this.serial_no_size, "utf_16_le"),  # serial_no is a (zero) padded string
+            "access_code_size" / Rebuild(Int32ub, len_(this.access_code) * 2),  # calculates the length of the access_code
+            "access_code" / PaddedString(this.access_code_size, "utf_16_le"),  # access_code is a (zero) padded string
+            "chip_id_size" / Rebuild(Int32ub, len_(this.chip_id) * 2),  # calculates the length of the chip_id
+            "chip_id" / PaddedString(this.chip_id_size, "utf_16_le"),  # chip_id is a (zero) padded string
+            "free_ticket_distribution_target_flag" / Int8ub,  # free_ticket_distribution_target_flag is a byte
+        )
+
+        req_data = req_struct.parse(req)
+        access_code = req_data.access_code
+
         user_id = self.core_data.card.get_user_id_from_card( access_code )
         profile_data = self.game_data.profile.get_profile(user_id)
 
@@ -118,7 +177,17 @@ class SaoBase:
 
     def handle_c500(self, request: Any) -> bytes:
         #user_info/get_user_basic_data
-        user_id = bytes.fromhex(request[88:112]).decode("utf-16le")
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+        )
+
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+
         profile_data = self.game_data.profile.get_profile(user_id)
 
         resp = SaoGetUserBasicDataResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, profile_data)
@@ -127,6 +196,7 @@ class SaoBase:
     def handle_c600(self, request: Any) -> bytes:
         #have_object/get_hero_log_user_data_list
         req = bytes.fromhex(request)[24:]
+        
         req_struct = Struct(
             Padding(16),
             "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
@@ -143,16 +213,38 @@ class SaoBase:
     
     def handle_c602(self, request: Any) -> bytes:
         #have_object/get_equipment_user_data_list
-        equipmentIdsData = self.game_data.static.get_equipment_ids(0, True)
-        
-        resp = SaoGetEquipmentUserDataListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, equipmentIdsData)
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+
+        )
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+    
+        equipment_data = self.game_data.item.get_user_equipments(user_id)
+
+        resp = SaoGetEquipmentUserDataListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, equipment_data)
         return resp.make()
         
     def handle_c604(self, request: Any) -> bytes:
         #have_object/get_item_user_data_list
-        itemIdsData = self.game_data.static.get_item_ids(0, True)
-        
-        resp = SaoGetItemUserDataListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, itemIdsData)
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+
+        )
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+
+        item_data = self.game_data.item.get_user_items(user_id)
+
+        resp = SaoGetItemUserDataListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, item_data)
         return resp.make()
         
     def handle_c606(self, request: Any) -> bytes:
@@ -171,7 +263,17 @@ class SaoBase:
         
     def handle_c608(self, request: Any) -> bytes:
         #have_object/get_episode_append_data_list
-        user_id = bytes.fromhex(request[88:112]).decode("utf-16le")
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+        )
+
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+
         profile_data = self.game_data.profile.get_profile(user_id)
 
         resp = SaoGetEpisodeAppendDataListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, profile_data)
@@ -179,8 +281,8 @@ class SaoBase:
 
     def handle_c804(self, request: Any) -> bytes:
         #custom/get_party_data_list
-
         req = bytes.fromhex(request)[24:]
+
         req_struct = Struct(
             Padding(16),
             "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
@@ -210,8 +312,20 @@ class SaoBase:
 
     def handle_c900(self, request: Any) -> bytes:
         #quest/get_quest_scene_user_data_list // QuestScene.csv
-        questIdsData = self.game_data.static.get_quests_ids(0, True)
-        resp = SaoGetQuestSceneUserDataListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, questIdsData)
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+
+        )
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+
+        quest_data = self.game_data.item.get_quest_logs(user_id)
+
+        resp = SaoGetQuestSceneUserDataListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, quest_data)
         return resp.make()
 
     def handle_c400(self, request: Any) -> bytes:
@@ -229,6 +343,159 @@ class SaoBase:
         resp = SaoCheckProfileCardUsedRewardResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
         return resp.make()
 
+    def handle_c814(self, request: Any) -> bytes:
+        #custom/synthesize_enhancement_hero_log
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(20),
+            "ticket_id" / Bytes(1),  # needs to be parsed as an int
+            Padding(1),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+            "origin_user_hero_log_id_size" / Rebuild(Int32ub, len_(this.origin_user_hero_log_id) * 2),  # calculates the length of the origin_user_hero_log_id
+            "origin_user_hero_log_id" / PaddedString(this.origin_user_hero_log_id_size, "utf_16_le"),  # origin_user_hero_log_id is a (zero) padded string
+            Padding(3),
+            "material_common_reward_user_data_list_length" / Rebuild(Int8ub, len_(this.material_common_reward_user_data_list)),  # material_common_reward_user_data_list is a byte,
+            "material_common_reward_user_data_list" / Array(this.material_common_reward_user_data_list_length, Struct(
+                "common_reward_type" / Int16ub,  # team_no is a byte
+                "user_common_reward_id_size" / Rebuild(Int32ub, len_(this.user_common_reward_id) * 2),  # calculates the length of the user_common_reward_id
+                "user_common_reward_id" / PaddedString(this.user_common_reward_id_size, "utf_16_le"),  # user_common_reward_id is a (zero) padded string
+            )),
+        )
+
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+        synthesize_hero_log_data = self.game_data.item.get_hero_log(req_data.user_id, req_data.origin_user_hero_log_id)
+
+        for i in range(0,req_data.material_common_reward_user_data_list_length):
+
+            itemList = self.game_data.static.get_item_id(req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+            heroList = self.game_data.static.get_hero_id(req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+            equipmentList = self.game_data.static.get_equipment_id(req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            if itemList:
+                hero_exp = 2000 + int(synthesize_hero_log_data["log_exp"])
+                self.game_data.item.remove_item(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            if equipmentList:
+                equipment_data = self.game_data.item.get_user_equipment(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+                hero_exp = int(equipment_data["enhancement_exp"]) + int(synthesize_hero_log_data["log_exp"])
+                self.game_data.item.remove_equipment(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            if heroList:
+                hero_data = self.game_data.item.get_hero_log(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+                hero_exp = int(hero_data["log_exp"]) + int(synthesize_hero_log_data["log_exp"])
+                self.game_data.item.remove_hero_log(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            self.game_data.item.put_hero_log(
+                user_id, 
+                int(req_data.origin_user_hero_log_id), 
+                synthesize_hero_log_data["log_level"], 
+                hero_exp, 
+                synthesize_hero_log_data["main_weapon"], 
+                synthesize_hero_log_data["sub_equipment"], 
+                synthesize_hero_log_data["skill_slot1_skill_id"], 
+                synthesize_hero_log_data["skill_slot2_skill_id"], 
+                synthesize_hero_log_data["skill_slot3_skill_id"], 
+                synthesize_hero_log_data["skill_slot4_skill_id"], 
+                synthesize_hero_log_data["skill_slot5_skill_id"]
+            )
+
+            profile = self.game_data.profile.get_profile(req_data.user_id)
+            new_col = int(profile["own_col"]) - 100
+
+            # Update profile
+            
+            self.game_data.profile.put_profile(
+                req_data.user_id,
+                profile["user_type"], 
+                profile["nick_name"], 
+                profile["rank_num"],
+                profile["rank_exp"],
+                new_col,
+                profile["own_vp"], 
+                profile["own_yui_medal"], 
+                profile["setting_title_id"]
+            )
+
+        # Load the item again to push to the response handler  
+        synthesize_hero_log_data = self.game_data.item.get_hero_log(req_data.user_id, req_data.origin_user_hero_log_id)
+
+        resp = SaoSynthesizeEnhancementHeroLogResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, synthesize_hero_log_data)
+        return resp.make()
+
+    def handle_c816(self, request: Any) -> bytes:
+        #custom/synthesize_enhancement_equipment
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(20),
+            "ticket_id" / Bytes(1),  # needs to be parsed as an int
+            Padding(1),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+            "origin_user_equipment_id_size" / Rebuild(Int32ub, len_(this.origin_user_equipment_id) * 2),  # calculates the length of the origin_user_equipment_id
+            "origin_user_equipment_id" / PaddedString(this.origin_user_equipment_id_size, "utf_16_le"),  # origin_user_equipment_id is a (zero) padded string
+            Padding(3),
+            "material_common_reward_user_data_list_length" / Rebuild(Int8ub, len_(this.material_common_reward_user_data_list)),  # material_common_reward_user_data_list is a byte,
+            "material_common_reward_user_data_list" / Array(this.material_common_reward_user_data_list_length, Struct(
+                "common_reward_type" / Int16ub,  # team_no is a byte
+                "user_common_reward_id_size" / Rebuild(Int32ub, len_(this.user_common_reward_id) * 2),  # calculates the length of the user_common_reward_id
+                "user_common_reward_id" / PaddedString(this.user_common_reward_id_size, "utf_16_le"),  # user_common_reward_id is a (zero) padded string
+            )),
+        )
+
+        req_data = req_struct.parse(req)
+
+        user_id = req_data.user_id
+        synthesize_equipment_data = self.game_data.item.get_user_equipment(req_data.user_id, req_data.origin_user_equipment_id)
+
+        for i in range(0,req_data.material_common_reward_user_data_list_length):
+
+            itemList = self.game_data.static.get_item_id(req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+            heroList = self.game_data.static.get_hero_id(req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+            equipmentList = self.game_data.static.get_equipment_id(req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            if itemList:
+                equipment_exp = 2000 + int(synthesize_equipment_data["enhancement_exp"])
+                self.game_data.item.remove_item(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            if equipmentList:
+                equipment_data = self.game_data.item.get_user_equipment(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+                equipment_exp = int(equipment_data["enhancement_exp"]) + int(synthesize_equipment_data["enhancement_exp"])
+                self.game_data.item.remove_equipment(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            if heroList:
+                hero_data = self.game_data.item.get_hero_log(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+                equipment_exp = int(hero_data["log_exp"]) + int(synthesize_equipment_data["enhancement_exp"])
+                self.game_data.item.remove_hero_log(req_data.user_id, req_data.material_common_reward_user_data_list[i].user_common_reward_id)
+
+            self.game_data.item.put_equipment_data(req_data.user_id, int(req_data.origin_user_equipment_id), synthesize_equipment_data["enhancement_value"], equipment_exp, 0, 0, 0)
+
+            profile = self.game_data.profile.get_profile(req_data.user_id)
+            new_col = int(profile["own_col"]) - 100
+
+            # Update profile
+            
+            self.game_data.profile.put_profile(
+                req_data.user_id,
+                profile["user_type"], 
+                profile["nick_name"], 
+                profile["rank_num"],
+                profile["rank_exp"],
+                new_col,
+                profile["own_vp"], 
+                profile["own_yui_medal"], 
+                profile["setting_title_id"]
+                )
+
+        # Load the item again to push to the response handler  
+        synthesize_equipment_data = self.game_data.item.get_user_equipment(req_data.user_id, req_data.origin_user_equipment_id)
+
+        resp = SaoSynthesizeEnhancementEquipmentResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, synthesize_equipment_data)
+        return resp.make()
+
     def handle_c806(self, request: Any) -> bytes:
         #custom/change_party
         req = bytes.fromhex(request)[24:]
@@ -270,6 +537,7 @@ class SaoBase:
 
         req_data = req_struct.parse(req)
         user_id = req_data.user_id
+        party_hero_list = []
 
         for party_team in req_data.party_data_list[0].party_team_data_list:
             hero_data = self.game_data.item.get_hero_log(user_id, party_team["user_hero_log_id"])
@@ -294,12 +562,15 @@ class SaoBase:
                 party_team["skill_slot5_skill_id"]
             )
 
+            party_hero_list.append(party_team["user_hero_log_id"])
+
+        self.game_data.item.put_hero_party(user_id, req_data.party_data_list[0].party_team_data_list[0].user_party_team_id, party_hero_list[0], party_hero_list[1], party_hero_list[2])
+
         resp = SaoNoopResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
         return resp.make()
 
     def handle_c904(self, request: Any) -> bytes:
         #quest/episode_play_start
-
         req = bytes.fromhex(request)[24:]
 
         req_struct = Struct(
@@ -432,8 +703,349 @@ class SaoBase:
 
         req_data = req_struct.parse(req)
 
+        # Add stage progression to database
+        user_id = req_data.user_id
+        episode_id = req_data.episode_id
+        quest_clear_flag = bool(req_data.score_data[0].boss_destroying_num)
+        clear_time = req_data.score_data[0].clear_time
+        combo_num = req_data.score_data[0].combo_num
+        total_damage = req_data.score_data[0].total_damage
+        concurrent_destroying_num = req_data.score_data[0].concurrent_destroying_num
+
+        profile = self.game_data.profile.get_profile(user_id)
+        vp = int(profile["own_vp"])
+        exp = int(profile["rank_exp"]) + 100 #always 100 extra exp for some reason
+        col = int(profile["own_col"]) + int(req_data.base_get_data[0].get_col)
+
+        if quest_clear_flag is True:
+            # Save stage progression - to be revised to avoid saving worse score
+
+            # Reference Episode.csv but Chapter 2,3,4 and 5 reports id -1, match using /10 + last digits
+            if episode_id > 10000 and episode_id < 11000:
+                # Starts at 1001
+                episode_id = episode_id - 9000
+            elif episode_id > 20000:
+                # Starts at 2001
+                stage_id = str(episode_id)[-2:]
+                episode_id = episode_id / 10
+                episode_id = int(episode_id) + int(stage_id)
+
+                # Match episode_id with the questSceneId saved in the DB through sortNo
+                questId = self.game_data.static.get_quests_id(episode_id)
+                episode_id = questId[2]
+
+            self.game_data.item.put_player_quest(user_id, episode_id, quest_clear_flag, clear_time, combo_num, total_damage, concurrent_destroying_num)
+
+            vp = int(profile["own_vp"]) + 10 #always 10 VP per cleared stage
+
+
+        # Calculate level based off experience and the CSV list
+        with open(r'titles/sao/data/PlayerRank.csv') as csv_file:
+            csv_reader = csv.reader(csv_file, delimiter=',')
+            line_count = 0
+            data = []
+            rowf = False
+            for row in csv_reader:
+                if rowf==False:
+                    rowf=True
+                else:
+                    data.append(row)           
+            
+        for i in range(0,len(data)):
+            if exp>=int(data[i][1]) and exp<int(data[i+1][1]):
+                player_level = int(data[i][0])
+                break
+
+        # Update profile
+        updated_profile = self.game_data.profile.put_profile(
+            user_id,
+            profile["user_type"], 
+            profile["nick_name"], 
+            player_level,
+            exp,
+            col,
+            vp, 
+            profile["own_yui_medal"], 
+            profile["setting_title_id"]
+            )
+
+        # Update heroes from the used party
+        play_session = self.game_data.item.get_session(user_id)
+        session_party = self.game_data.item.get_hero_party(user_id, play_session["user_party_team_id"])
+
+        hero_list = []
+        hero_list.append(session_party["user_hero_log_id_1"])
+        hero_list.append(session_party["user_hero_log_id_2"])
+        hero_list.append(session_party["user_hero_log_id_3"])
+
+        for i in range(0,len(hero_list)):
+            hero_data = self.game_data.item.get_hero_log(user_id, hero_list[i])
+
+            log_exp = int(hero_data["log_exp"]) + int(req_data.base_get_data[0].get_hero_log_exp)
+
+            # Calculate hero level based off experience and the CSV list
+            with open(r'titles/sao/data/HeroLogLevel.csv') as csv_file:
+                csv_reader = csv.reader(csv_file, delimiter=',')
+                line_count = 0
+                data = []
+                rowf = False
+                for row in csv_reader:
+                    if rowf==False:
+                        rowf=True
+                    else:
+                        data.append(row)
+                
+            for e in range(0,len(data)):
+                if log_exp>=int(data[e][1]) and log_exp<int(data[e+1][1]):
+                    hero_level = int(data[e][0])
+                    break
+
+            self.game_data.item.put_hero_log(
+                user_id,
+                hero_data["user_hero_log_id"],
+                hero_level,
+                log_exp,
+                hero_data["main_weapon"],
+                hero_data["sub_equipment"],
+                hero_data["skill_slot1_skill_id"],
+                hero_data["skill_slot2_skill_id"],
+                hero_data["skill_slot3_skill_id"],
+                hero_data["skill_slot4_skill_id"],
+                hero_data["skill_slot5_skill_id"]
+            )
+
+        # Grab the rare loot from the table, match it with the right item and then push to the player profile
+        json_data = {"data": []}
+
+        for r in range(0,req_data.get_rare_drop_data_list_length):
+            rewardList = self.game_data.static.get_rare_drop_id(int(req_data.get_rare_drop_data_list[r].quest_rare_drop_id))
+            commonRewardId = rewardList["commonRewardId"]
+
+            heroList = self.game_data.static.get_hero_id(commonRewardId)
+            equipmentList = self.game_data.static.get_equipment_id(commonRewardId)
+            itemList = self.game_data.static.get_item_id(commonRewardId)
+
+            if heroList:
+                self.game_data.item.put_hero_log(user_id, commonRewardId, 1, 0, 101000016, 0, 30086, 1001, 1002, 0, 0)
+            if equipmentList:
+                self.game_data.item.put_equipment_data(user_id, commonRewardId, 1, 200, 0, 0, 0)
+            if itemList:
+                self.game_data.item.put_item(user_id, commonRewardId)
+        
+        # Generate random hero(es) based off the response    
+        for a in range(0,req_data.get_unanalyzed_log_tmp_reward_data_list_length):
+            
+            with open('titles/sao/data/RewardTable.csv', 'r') as f:
+                keys_unanalyzed = next(f).strip().split(',')
+                data_unanalyzed = list(DictReader(f, fieldnames=keys_unanalyzed))
+
+            randomized_unanalyzed_id = choice(data_unanalyzed)
+            while int(randomized_unanalyzed_id['UnanalyzedLogGradeId']) != req_data.get_unanalyzed_log_tmp_reward_data_list[a].unanalyzed_log_grade_id:
+                randomized_unanalyzed_id = choice(data_unanalyzed)
+            
+            heroList = self.game_data.static.get_hero_id(randomized_unanalyzed_id['CommonRewardId'])
+            equipmentList = self.game_data.static.get_equipment_id(randomized_unanalyzed_id['CommonRewardId'])
+            itemList = self.game_data.static.get_item_id(randomized_unanalyzed_id['CommonRewardId'])
+            if heroList:
+                self.game_data.item.put_hero_log(user_id, randomized_unanalyzed_id['CommonRewardId'], 1, 0, 101000016, 0, 30086, 1001, 1002, 0, 0)
+            if equipmentList:
+                self.game_data.item.put_equipment_data(user_id, randomized_unanalyzed_id['CommonRewardId'], 1, 200, 0, 0, 0)
+            if itemList:
+                self.game_data.item.put_item(user_id, randomized_unanalyzed_id['CommonRewardId'])
+
+            json_data["data"].append(randomized_unanalyzed_id['CommonRewardId'])
+            
+        # Send response
+
+        self.game_data.item.create_end_session(user_id, episode_id, quest_clear_flag, json_data["data"])
+
+        resp = SaoEpisodePlayEndResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        return resp.make()
+
+    def handle_c914(self, request: Any) -> bytes:
+        #quest/trial_tower_play_start
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "ticket_id_size" / Rebuild(Int32ub, len_(this.ticket_id) * 2),  # calculates the length of the ticket_id
+            "ticket_id" / PaddedString(this.ticket_id_size, "utf_16_le"),  # ticket_id is a (zero) padded string
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+            "trial_tower_id" / Int32ub,  # trial_tower_id is an int
+            "play_mode" / Int8ub,  # play_mode is a byte
+            Padding(3),
+            "play_start_request_data_length" / Rebuild(Int8ub, len_(this.play_start_request_data)),  # play_start_request_data_length is a byte,
+            "play_start_request_data" / Array(this.play_start_request_data_length, Struct(
+                "user_party_id_size" / Rebuild(Int32ub, len_(this.user_party_id) * 2),  # calculates the length of the user_party_id
+                "user_party_id" / PaddedString(this.user_party_id_size, "utf_16_le"),  # user_party_id is a (zero) padded string
+                "appoint_leader_resource_card_code_size" / Rebuild(Int32ub, len_(this.appoint_leader_resource_card_code) * 2),  # calculates the length of the total_damage
+                "appoint_leader_resource_card_code" / PaddedString(this.appoint_leader_resource_card_code_size, "utf_16_le"),  # total_damage is a (zero) padded string
+                "use_profile_card_code_size" / Rebuild(Int32ub, len_(this.use_profile_card_code) * 2),  # calculates the length of the total_damage
+                "use_profile_card_code" / PaddedString(this.use_profile_card_code_size, "utf_16_le"),  # use_profile_card_code is a (zero) padded string
+                "quest_drop_boost_apply_flag" / Int8ub,  # quest_drop_boost_apply_flag is a byte
+            )),
+        )
+
+        req_data = req_struct.parse(req)
+
+        user_id = req_data.user_id
+        floor_id = req_data.trial_tower_id
+        profile_data = self.game_data.profile.get_profile(user_id)
+
+        self.game_data.item.create_session(
+            user_id, 
+            int(req_data.play_start_request_data[0].user_party_id), 
+            req_data.trial_tower_id, 
+            req_data.play_mode, 
+            req_data.play_start_request_data[0].quest_drop_boost_apply_flag
+            )
+
+        resp = SaoEpisodePlayStartResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, profile_data)
+        return resp.make()
+
+    def handle_c918(self, request: Any) -> bytes:
+        #quest/trial_tower_play_end
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(20),
+            "ticket_id" / Bytes(1),  # needs to be parsed as an int
+            Padding(1),
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+            Padding(2),
+            "trial_tower_id" / Int16ub,  # trial_tower_id is a short,
+            Padding(3),
+            "play_end_request_data" / Int8ub,  # play_end_request_data is a byte
+            Padding(1),
+            "play_result_flag" / Int8ub,  # play_result_flag is a byte
+            Padding(2),
+            "base_get_data_length" / Rebuild(Int8ub, len_(this.base_get_data)),  # base_get_data_length is a byte,
+            "base_get_data" / Array(this.base_get_data_length, Struct(
+                "get_hero_log_exp" / Int32ub,  # get_hero_log_exp is an int
+                "get_col" / Int32ub,  # get_num is a short
+            )),
+            Padding(3),
+            "get_player_trace_data_list_length" / Rebuild(Int8ub, len_(this.get_player_trace_data_list)),  # get_player_trace_data_list_length is a byte
+            "get_player_trace_data_list" / Array(this.get_player_trace_data_list_length, Struct(
+                "user_quest_scene_player_trace_id" / Int32ub,  # user_quest_scene_player_trace_id is an int
+            )),
+            Padding(3),
+            "get_rare_drop_data_list_length" / Rebuild(Int8ub, len_(this.get_rare_drop_data_list)),  # get_rare_drop_data_list_length is a byte
+            "get_rare_drop_data_list" / Array(this.get_rare_drop_data_list_length, Struct(
+                "quest_rare_drop_id" / Int32ub,  # quest_rare_drop_id is an int
+            )),
+            Padding(3),
+            "get_special_rare_drop_data_list_length" / Rebuild(Int8ub, len_(this.get_special_rare_drop_data_list)),  # get_special_rare_drop_data_list_length is a byte
+            "get_special_rare_drop_data_list" / Array(this.get_special_rare_drop_data_list_length, Struct(
+                "quest_special_rare_drop_id" / Int32ub,  # quest_special_rare_drop_id is an int
+            )),
+            Padding(3),
+            "get_unanalyzed_log_tmp_reward_data_list_length" / Rebuild(Int8ub, len_(this.get_unanalyzed_log_tmp_reward_data_list)),  # get_unanalyzed_log_tmp_reward_data_list_length is a byte
+            "get_unanalyzed_log_tmp_reward_data_list" / Array(this.get_unanalyzed_log_tmp_reward_data_list_length, Struct(
+                "unanalyzed_log_grade_id" / Int32ub,  # unanalyzed_log_grade_id is an int,
+            )),
+            Padding(3),
+            "get_event_item_data_list_length" / Rebuild(Int8ub, len_(this.get_event_item_data_list)),  # get_event_item_data_list_length is a byte,
+            "get_event_item_data_list" / Array(this.get_event_item_data_list_length, Struct(
+                "event_item_id" / Int32ub,  # event_item_id is an int
+                "get_num" / Int16ub,  # get_num is a short
+            )),
+            Padding(3),
+            "discovery_enemy_data_list_length" / Rebuild(Int8ub, len_(this.discovery_enemy_data_list)),  # discovery_enemy_data_list_length is a byte
+            "discovery_enemy_data_list" / Array(this.discovery_enemy_data_list_length, Struct(
+                "enemy_kind_id" / Int32ub,  # enemy_kind_id is an int
+                "destroy_num" / Int16ub,  # destroy_num is a short
+            )),
+            Padding(3),
+            "destroy_boss_data_list_length" / Rebuild(Int8ub, len_(this.destroy_boss_data_list)),  # destroy_boss_data_list_length is a byte
+            "destroy_boss_data_list" / Array(this.destroy_boss_data_list_length, Struct(
+                "boss_type" / Int8ub,  # boss_type is a byte
+                "enemy_kind_id" / Int32ub,  # enemy_kind_id is an int
+                "destroy_num" / Int16ub,  # destroy_num is a short
+            )),
+            Padding(3),
+            "mission_data_list_length" / Rebuild(Int8ub, len_(this.mission_data_list)),  # mission_data_list_length is a byte
+            "mission_data_list" / Array(this.mission_data_list_length, Struct(
+                "mission_id" / Int32ub,  # enemy_kind_id is an int
+                "clear_flag" / Int8ub,  # boss_type is a byte
+                "mission_difficulty_id" / Int16ub,  # destroy_num is a short
+            )),
+            Padding(3),
+            "score_data_length" / Rebuild(Int8ub, len_(this.score_data)),  # score_data_length is a byte
+            "score_data" / Array(this.score_data_length, Struct(
+                "clear_time" / Int32ub,  # clear_time is an int
+                "combo_num" / Int32ub,  # boss_type is a int
+                "total_damage_size" / Rebuild(Int32ub, len_(this.total_damage) * 2),  # calculates the length of the total_damage
+                "total_damage" / PaddedString(this.total_damage_size, "utf_16_le"),  # total_damage is a (zero) padded string
+                "concurrent_destroying_num" / Int16ub, # concurrent_destroying_num is a short
+                "reaching_skill_level" / Int16ub, # reaching_skill_level is a short
+                "ko_chara_num" / Int8ub,  # ko_chara_num is a byte
+                "acceleration_invocation_num" / Int16ub, # acceleration_invocation_num is a short
+                "boss_destroying_num" / Int16ub, # boss_destroying_num is a short
+                "synchro_skill_used_flag" / Int8ub,  # synchro_skill_used_flag is a byte
+                "used_friend_skill_id" / Int32ub,  # used_friend_skill_id is an int
+                "friend_skill_used_flag" / Int8ub,  # friend_skill_used_flag is a byte
+                "continue_cnt" / Int16ub, # continue_cnt is a short
+                "total_loss_num" / Int16ub, # total_loss_num is a short
+            )),
+
+        )
+
+        req_data = req_struct.parse(req)
+
+        # Add tower progression to database
+        user_id = req_data.user_id
+        trial_tower_id = req_data.trial_tower_id
+        next_tower_id = 0
+        quest_clear_flag = bool(req_data.score_data[0].boss_destroying_num)
+        clear_time = req_data.score_data[0].clear_time
+        combo_num = req_data.score_data[0].combo_num
+        total_damage = req_data.score_data[0].total_damage
+        concurrent_destroying_num = req_data.score_data[0].concurrent_destroying_num
+
+        if quest_clear_flag is True:
+            # Save tower progression - to be revised to avoid saving worse score
+            if trial_tower_id == 9: 
+                next_tower_id = 10001
+            elif trial_tower_id == 10: 
+                trial_tower_id = 10001
+                next_tower_id = 3011
+            elif trial_tower_id == 19: 
+                next_tower_id = 10002
+            elif trial_tower_id == 20:
+                trial_tower_id = 10002
+                next_tower_id = 3021
+            elif trial_tower_id == 29: 
+                next_tower_id = 10003
+            elif trial_tower_id == 30:
+                trial_tower_id = 10003
+                next_tower_id = 3031
+            elif trial_tower_id == 39: 
+                next_tower_id = 10004
+            elif trial_tower_id == 40:
+                trial_tower_id = 10004
+                next_tower_id = 3041
+            elif trial_tower_id == 49: 
+                next_tower_id = 10005
+            elif trial_tower_id == 50:
+                trial_tower_id = 10005
+                next_tower_id = 3051
+            else:
+                trial_tower_id = trial_tower_id + 3000
+                next_tower_id = trial_tower_id + 1
+
+            self.game_data.item.put_player_quest(user_id, trial_tower_id, quest_clear_flag, clear_time, combo_num, total_damage, concurrent_destroying_num)
+
+            # Check if next stage is already done
+            checkQuest = self.game_data.item.get_quest_log(user_id, next_tower_id)
+            if not checkQuest:
+                if next_tower_id != 3101:
+                    self.game_data.item.put_player_quest(user_id, next_tower_id, 0, 0, 0, 0, 0)
+
         # Update the profile 
-        profile = self.game_data.profile.get_profile(req_data.user_id)
+        profile = self.game_data.profile.get_profile(user_id)
         
         exp = int(profile["rank_exp"]) + 100 #always 100 extra exp for some reason
         col = int(profile["own_col"]) + int(req_data.base_get_data[0].get_col)
@@ -455,9 +1067,8 @@ class SaoBase:
                 player_level = int(data[i][0])
                 break
 
-        # Update profile
         updated_profile = self.game_data.profile.put_profile(
-            req_data.user_id,
+            user_id,
             profile["user_type"], 
             profile["nick_name"], 
             player_level,
@@ -469,8 +1080,8 @@ class SaoBase:
             )
 
         # Update heroes from the used party
-        play_session = self.game_data.item.get_session(req_data.user_id)
-        session_party = self.game_data.item.get_hero_party(req_data.user_id, play_session["user_party_team_id"])
+        play_session = self.game_data.item.get_session(user_id)
+        session_party = self.game_data.item.get_hero_party(user_id, play_session["user_party_team_id"])
 
         hero_list = []
         hero_list.append(session_party["user_hero_log_id_1"])
@@ -478,14 +1089,31 @@ class SaoBase:
         hero_list.append(session_party["user_hero_log_id_3"])
 
         for i in range(0,len(hero_list)):
-            hero_data = self.game_data.item.get_hero_log(req_data.user_id, hero_list[i])
+            hero_data = self.game_data.item.get_hero_log(user_id, hero_list[i])
 
             log_exp = int(hero_data["log_exp"]) + int(req_data.base_get_data[0].get_hero_log_exp)
 
+            # Calculate hero level based off experience and the CSV list
+            with open(r'titles/sao/data/HeroLogLevel.csv') as csv_file:
+                csv_reader = csv.reader(csv_file, delimiter=',')
+                line_count = 0
+                data = []
+                rowf = False
+                for row in csv_reader:
+                    if rowf==False:
+                        rowf=True
+                    else:
+                        data.append(row)
+                
+            for e in range(0,len(data)):
+                if log_exp>=int(data[e][1]) and log_exp<int(data[e+1][1]):
+                    hero_level = int(data[e][0])
+                    break
+
             self.game_data.item.put_hero_log(
-                req_data.user_id,
+                user_id,
                 hero_data["user_hero_log_id"],
-                hero_data["log_level"],
+                hero_level,
                 log_exp,
                 hero_data["main_weapon"],
                 hero_data["sub_equipment"],
@@ -496,19 +1124,121 @@ class SaoBase:
                 hero_data["skill_slot5_skill_id"]
             )
 
-        resp = SaoEpisodePlayEndResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        json_data = {"data": []}
+        
+        # Grab the rare loot from the table, match it with the right item and then push to the player profile
+        for r in range(0,req_data.get_rare_drop_data_list_length):
+            rewardList = self.game_data.static.get_rare_drop_id(int(req_data.get_rare_drop_data_list[r].quest_rare_drop_id))
+            commonRewardId = rewardList["commonRewardId"]
+
+            heroList = self.game_data.static.get_hero_id(commonRewardId)
+            equipmentList = self.game_data.static.get_equipment_id(commonRewardId)
+            itemList = self.game_data.static.get_item_id(commonRewardId)
+
+            if heroList:
+                self.game_data.item.put_hero_log(user_id, commonRewardId, 1, 0, 101000016, 0, 30086, 1001, 1002, 0, 0)
+            if equipmentList:
+                self.game_data.item.put_equipment_data(user_id, commonRewardId, 1, 200, 0, 0, 0)
+            if itemList:
+                self.game_data.item.put_item(user_id, commonRewardId)
+
+        # Generate random hero(es) based off the response    
+        for a in range(0,req_data.get_unanalyzed_log_tmp_reward_data_list_length):
+            
+            with open('titles/sao/data/RewardTable.csv', 'r') as f:
+                keys_unanalyzed = next(f).strip().split(',')
+                data_unanalyzed = list(DictReader(f, fieldnames=keys_unanalyzed))
+
+            randomized_unanalyzed_id = choice(data_unanalyzed)
+            while int(randomized_unanalyzed_id['UnanalyzedLogGradeId']) != req_data.get_unanalyzed_log_tmp_reward_data_list[a].unanalyzed_log_grade_id:
+                randomized_unanalyzed_id = choice(data_unanalyzed)
+            
+            heroList = self.game_data.static.get_hero_id(randomized_unanalyzed_id['CommonRewardId'])
+            equipmentList = self.game_data.static.get_equipment_id(randomized_unanalyzed_id['CommonRewardId'])
+            itemList = self.game_data.static.get_item_id(randomized_unanalyzed_id['CommonRewardId'])
+            if heroList:
+                self.game_data.item.put_hero_log(user_id, randomized_unanalyzed_id['CommonRewardId'], 1, 0, 101000016, 0, 30086, 1001, 1002, 0, 0)
+            if equipmentList:
+                self.game_data.item.put_equipment_data(user_id, randomized_unanalyzed_id['CommonRewardId'], 1, 200, 0, 0, 0)
+            if itemList:
+                self.game_data.item.put_item(user_id, randomized_unanalyzed_id['CommonRewardId'])
+
+            json_data["data"].append(randomized_unanalyzed_id['CommonRewardId'])
+            
+        # Send response
+
+        self.game_data.item.create_end_session(user_id, trial_tower_id, quest_clear_flag, json_data["data"])
+
+        resp = SaoTrialTowerPlayEndResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
         return resp.make()
 
-    def handle_c914(self, request: Any) -> bytes:
-        #quest/trial_tower_play_start
-        user_id = bytes.fromhex(request[100:124]).decode("utf-16le")
-        floor_id = int(request[130:132], 16) # not required but nice to know
-        profile_data = self.game_data.profile.get_profile(user_id)
-
-        resp = SaoEpisodePlayStartResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, profile_data)
-        return resp.make()
-
-    def handle_c90a(self, request: Any) -> bytes: #should be tweaked for proper item unlock
+    def handle_c90a(self, request: Any) -> bytes:
         #quest/episode_play_end_unanalyzed_log_fixed
-        resp = SaoEpisodePlayEndUnanalyzedLogFixedResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "ticket_id_size" / Rebuild(Int32ub, len_(this.ticket_id) * 2),  # calculates the length of the ticket_id
+            "ticket_id" / PaddedString(this.ticket_id_size, "utf_16_le"),  # ticket_id is a (zero) padded string
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+        )
+
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+
+        end_session_data = self.game_data.item.get_end_session(user_id)
+
+        resp = SaoEpisodePlayEndUnanalyzedLogFixedResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, end_session_data[4])
         return resp.make()
+
+    def handle_c91a(self, request: Any) -> bytes: # handler is identical to the episode
+        #quest/trial_tower_play_end_unanalyzed_log_fixed
+        req = bytes.fromhex(request)[24:]
+
+        req_struct = Struct(
+            Padding(16),
+            "ticket_id_size" / Rebuild(Int32ub, len_(this.ticket_id) * 2),  # calculates the length of the ticket_id
+            "ticket_id" / PaddedString(this.ticket_id_size, "utf_16_le"),  # ticket_id is a (zero) padded string
+            "user_id_size" / Rebuild(Int32ub, len_(this.user_id) * 2),  # calculates the length of the user_id
+            "user_id" / PaddedString(this.user_id_size, "utf_16_le"),  # user_id is a (zero) padded string
+        )
+
+        req_data = req_struct.parse(req)
+        user_id = req_data.user_id
+
+        end_session_data = self.game_data.item.get_end_session(user_id)
+
+        resp = SaoEpisodePlayEndUnanalyzedLogFixedResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1, end_session_data[4])
+        return resp.make()
+
+    def handle_cd00(self, request: Any) -> bytes:
+        #defrag_match/get_defrag_match_basic_data
+        resp = SaoGetDefragMatchBasicDataResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        return resp.make()
+
+    def handle_cd02(self, request: Any) -> bytes:
+        #defrag_match/get_defrag_match_ranking_user_data
+        resp = SaoGetDefragMatchRankingUserDataResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        return resp.make()
+
+    def handle_cd04(self, request: Any) -> bytes:
+        #defrag_match/get_defrag_match_league_point_ranking_list
+        resp = SaoGetDefragMatchLeaguePointRankingListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        return resp.make()
+
+    def handle_cd06(self, request: Any) -> bytes:
+        #defrag_match/get_defrag_match_league_score_ranking_list
+        resp = SaoGetDefragMatchLeagueScoreRankingListResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        return resp.make()
+
+    def handle_d404(self, request: Any) -> bytes:
+        #other/bnid_serial_code_check
+        resp = SaoBnidSerialCodeCheckResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        return resp.make()
+
+    def handle_c306(self, request: Any) -> bytes:
+        #card/scan_qr_quest_profile_card
+        resp = SaoScanQrQuestProfileCardResponse(int.from_bytes(bytes.fromhex(request[:4]), "big")+1)
+        return resp.make()

titles/sao/data/EquipmentLevel.csv

@@ -0,0 +1,121 @@
+EquipmentLevelId,RequireExp
+1,200,
+2,400,
+3,600,
+4,800,
+5,1000,
+6,1200,
+7,1400,
+8,1600,
+9,1800,
+10,2000,
+11,2200,
+12,2400,
+13,2600,
+14,2800,
+15,3000,
+16,3200,
+17,3400,
+18,3600,
+19,3800,
+20,4000,
+21,4200,
+22,4400,
+23,4600,
+24,4800,
+25,5000,
+26,5200,
+27,5400,
+28,5600,
+29,5800,
+30,6000,
+31,6200,
+32,6400,
+33,6600,
+34,6800,
+35,7000,
+36,7200,
+37,7400,
+38,7600,
+39,7800,
+40,8000,
+41,8200,
+42,8400,
+43,8600,
+44,8800,
+45,9000,
+46,9200,
+47,9400,
+48,9600,
+49,9800,
+50,10000,
+51,10200,
+52,10400,
+53,10600,
+54,10800,
+55,11000,
+56,11200,
+57,11400,
+58,11600,
+59,11800,
+60,12000,
+61,12200,
+62,12400,
+63,12600,
+64,12800,
+65,13000,
+66,13200,
+67,13400,
+68,13600,
+69,13800,
+70,14000,
+71,14200,
+72,14400,
+73,14600,
+74,14800,
+75,15000,
+76,15200,
+77,15400,
+78,15600,
+79,15800,
+80,16000,
+81,16200,
+82,16400,
+83,16600,
+84,16800,
+85,17000,
+86,17200,
+87,17400,
+88,17600,
+89,17800,
+90,18000,
+91,18200,
+92,18400,
+93,18600,
+94,18800,
+95,19000,
+96,19200,
+97,19400,
+98,19600,
+99,19800,
+100,100000,
+101,150000,
+102,200000,
+103,250000,
+104,300000,
+105,350000,
+106,400000,
+107,450000,
+108,500000,
+109,550000,
+110,600000,
+111,650000,
+112,700000,
+113,750000,
+114,800000,
+115,850000,
+116,900000,
+117,950000,
+118,1000000,
+119,1000000,
+120,1000000,

titles/sao/database.py

@@ -8,5 +8,6 @@ class SaoData(Data):
     def __init__(self, cfg: CoreConfig) -> None:
         super().__init__(cfg)
 
+        self.item = SaoItemData(cfg, self.session)
         self.profile = SaoProfileData(cfg, self.session)
         self.static = SaoStaticData(cfg, self.session)

titles/sao/index.py

@@ -97,21 +97,21 @@ class SaoServlet(resource.Resource):
         req_url = request.uri.decode()
         if req_url == "/matching":
             self.logger.info("Matching request")
-
+        
         request.responseHeaders.addRawHeader(b"content-type", b"text/html; charset=utf-8")
 
         sao_request = request.content.getvalue().hex()
-        #sao_request = sao_request[:32]
 
         handler = getattr(self.base, f"handle_{sao_request[:4]}", None)
         if handler is None:
             self.logger.info(f"Generic Handler for {req_url} - {sao_request[:4]}")
-            #self.logger.debug(f"Request: {request.content.getvalue().hex()}")
+            self.logger.debug(f"Request: {request.content.getvalue().hex()}")
             resp = SaoNoopResponse(int.from_bytes(bytes.fromhex(sao_request[:4]), "big")+1)
             self.logger.debug(f"Response: {resp.make().hex()}")
             return resp.make()
 
         self.logger.info(f"Handler {req_url} - {sao_request[:4]} request")
         self.logger.debug(f"Request: {request.content.getvalue().hex()}")
-        self.logger.debug(f"Response: {handler(sao_request).hex()}")
-        return handler(sao_request)
+        resp = handler(sao_request)
+        self.logger.debug(f"Response: {resp.hex()}")
+        return resp

titles/sao/read.py

@@ -228,3 +228,27 @@ class SaoReader(BaseReader):
                         continue
         except:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
+
+        self.logger.info("Now reading RareDropTable.csv")
+        try:
+            fullPath = bin_dir + "/RareDropTable.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    questRareDropId = row["QuestRareDropId"]
+                    commonRewardId = row["CommonRewardId"]
+                    enabled = True
+
+                    self.logger.info(f"Added rare drop {questRareDropId} | Reward: {commonRewardId}")
+                    
+                    try:
+                        self.data.static.put_rare_drop(
+                            0,
+                            questRareDropId,
+                            commonRewardId,
+                            enabled
+                        )
+                    except Exception as err:
+                        print(err)
+        except:
+            self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")

titles/sao/schema/item.py

@@ -1,6 +1,6 @@
 from typing import Optional, Dict, List
 from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, case
-from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean
+from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select, update, delete
 from sqlalchemy.engine import Row
@@ -8,6 +8,41 @@ from sqlalchemy.dialects.mysql import insert
 
 from core.data.schema import BaseData, metadata
 
+equipment_data = Table(
+    "sao_equipment_data",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("equipment_id", Integer, nullable=False),
+    Column("enhancement_value", Integer, nullable=False),
+    Column("enhancement_exp", Integer, nullable=False),
+    Column("awakening_exp", Integer, nullable=False),
+    Column("awakening_stage", Integer, nullable=False),
+    Column("possible_awakening_flag", Integer, nullable=False),
+    Column("get_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "equipment_id", name="sao_equipment_data_uk"),
+    mysql_charset="utf8mb4",
+)
+
+item_data = Table(
+    "sao_item_data",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("item_id", Integer, nullable=False),
+    Column("get_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "item_id", name="sao_item_data_uk"),
+    mysql_charset="utf8mb4",
+)
+
 hero_log_data = Table(
     "sao_hero_log_data",
     metadata,
@@ -49,6 +84,26 @@ hero_party = Table(
     mysql_charset="utf8mb4",
 )
 
+quest = Table(
+    "sao_player_quest",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("episode_id", Integer, nullable=False),
+    Column("quest_clear_flag", Boolean, nullable=False),
+    Column("clear_time", Integer, nullable=False),
+    Column("combo_num", Integer, nullable=False),
+    Column("total_damage", Integer, nullable=False),
+    Column("concurrent_destroying_num", Integer, nullable=False),
+    Column("play_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "episode_id", name="sao_player_quest_uk"),
+    mysql_charset="utf8mb4",
+)
+
 sessions = Table(
     "sao_play_sessions",
     metadata,
@@ -67,6 +122,22 @@ sessions = Table(
     mysql_charset="utf8mb4",
 )
 
+end_sessions = Table(
+    "sao_end_sessions",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("quest_id", Integer, nullable=False),
+    Column("play_result_flag", Boolean, nullable=False),
+    Column("reward_data", JSON, nullable=True),
+    Column("play_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    mysql_charset="utf8mb4",
+)
+
 class SaoItemData(BaseData):
     def create_session(self, user_id: int, user_party_team_id: int, episode_id: int, play_mode: int, quest_drop_boost_apply_flag: int) -> Optional[int]:
         sql = insert(sessions).values(
@@ -85,6 +156,69 @@ class SaoItemData(BaseData):
             return None
         return result.lastrowid
 
+    def create_end_session(self, user_id: int, quest_id: int, play_result_flag: bool, reward_data: JSON) -> Optional[int]:
+        sql = insert(end_sessions).values(
+            user=user_id,
+            quest_id=quest_id,
+            play_result_flag=play_result_flag,
+            reward_data=reward_data,
+            )
+
+        conflict = sql.on_duplicate_key_update(user=user_id)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to create SAO end session for user {user_id}!")
+            return None
+        return result.lastrowid
+
+    def put_item(self, user_id: int, item_id: int) -> Optional[int]:
+        sql = insert(item_data).values(
+            user=user_id,
+            item_id=item_id,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            item_id=item_id,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert item! user: {user_id}, item_id: {item_id}"
+            )
+            return None
+
+        return result.lastrowid
+    
+    def put_equipment_data(self, user_id: int, equipment_id: int, enhancement_value: int, enhancement_exp: int, awakening_exp: int, awakening_stage: int, possible_awakening_flag: int) -> Optional[int]:
+        sql = insert(equipment_data).values(
+            user=user_id,
+            equipment_id=equipment_id,
+            enhancement_value=enhancement_value,
+            enhancement_exp=enhancement_exp,
+            awakening_exp=awakening_exp,
+            awakening_stage=awakening_stage,
+            possible_awakening_flag=possible_awakening_flag,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            enhancement_value=enhancement_value,
+            enhancement_exp=enhancement_exp,
+            awakening_exp=awakening_exp,
+            awakening_stage=awakening_stage,
+            possible_awakening_flag=possible_awakening_flag,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert equipment! user: {user_id}, equipment_id: {equipment_id}"
+            )
+            return None
+
+        return result.lastrowid
+
     def put_hero_log(self, user_id: int, user_hero_log_id: int, log_level: int, log_exp: int, main_weapon: int, sub_equipment: int, skill_slot1_skill_id: int, skill_slot2_skill_id: int, skill_slot3_skill_id: int, skill_slot4_skill_id: int, skill_slot5_skill_id: int) -> Optional[int]:
         sql = insert(hero_log_data).values(
             user=user_id,
@@ -145,6 +279,76 @@ class SaoItemData(BaseData):
 
         return result.lastrowid
 
+    def put_player_quest(self, user_id: int, episode_id: int, quest_clear_flag: bool, clear_time: int, combo_num: int, total_damage: int, concurrent_destroying_num: int) -> Optional[int]:
+        sql = insert(quest).values(
+            user=user_id,
+            episode_id=episode_id,
+            quest_clear_flag=quest_clear_flag,
+            clear_time=clear_time,
+            combo_num=combo_num,
+            total_damage=total_damage,
+            concurrent_destroying_num=concurrent_destroying_num
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            quest_clear_flag=quest_clear_flag,
+            clear_time=clear_time,
+            combo_num=combo_num,
+            total_damage=total_damage,
+            concurrent_destroying_num=concurrent_destroying_num
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert quest! user: {user_id}, episode_id: {episode_id}"
+            )
+            return None
+
+        return result.lastrowid
+
+    def get_user_equipment(self, user_id: int, equipment_id: int) -> Optional[Dict]:
+        sql = equipment_data.select(equipment_data.c.user == user_id and equipment_data.c.equipment_id == equipment_id)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def get_user_equipments(
+        self, user_id: int
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all equipments lookup given a profile
+        """
+        sql = equipment_data.select(
+            and_(
+                equipment_data.c.user == user_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_user_items(
+        self, user_id: int
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all items lookup given a profile
+        """
+        sql = item_data.select(
+            and_(
+                item_data.c.user == user_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_hero_log(
         self, user_id: int, user_hero_log_id: int = None
     ) -> Optional[List[Row]]:
@@ -167,7 +371,7 @@ class SaoItemData(BaseData):
         self, user_id: int
     ) -> Optional[List[Row]]:
         """
-        A catch-all hero lookup given a profile and user_party_team_id and ID specifiers
+        A catch-all hero lookup given a profile
         """
         sql = hero_log_data.select(
             and_(
@@ -195,6 +399,41 @@ class SaoItemData(BaseData):
             return None
         return result.fetchone()
 
+    def get_quest_log(
+        self, user_id: int, episode_id: int = None
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all quest lookup given a profile and episode_id
+        """
+        sql = quest.select(
+            and_(
+                quest.c.user == user_id,
+                quest.c.episode_id == episode_id if episode_id is not None else True,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_quest_logs(
+        self, user_id: int
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all quest lookup given a profile
+        """
+        sql = quest.select(
+            and_(
+                quest.c.user == user_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_session(
         self, user_id: int = None
     ) -> Optional[List[Row]]:
@@ -209,4 +448,59 @@ class SaoItemData(BaseData):
         result = self.execute(sql)
         if result is None:
             return None
-        return result.fetchone()
+        return result.fetchone()
+
+    def get_end_session(
+        self, user_id: int = None
+    ) -> Optional[List[Row]]:
+        sql = end_sessions.select(
+            and_(
+                end_sessions.c.user == user_id,
+            )
+        ).order_by(
+            end_sessions.c.play_date.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def remove_hero_log(self, user_id: int, user_hero_log_id: int) -> None:
+        sql = hero_log_data.delete(
+           and_(
+                hero_log_data.c.user == user_id,
+                hero_log_data.c.user_hero_log_id == user_hero_log_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to remove hero log! profile: {user_id}, user_hero_log_id: {user_hero_log_id}"
+            )
+        return None
+
+    def remove_equipment(self, user_id: int, equipment_id: int) -> None:
+        sql = equipment_data.delete(
+            and_(equipment_data.c.user == user_id, equipment_data.c.equipment_id == equipment_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to remove equipment! profile: {user_id}, equipment_id: {equipment_id}"
+            )
+        return None
+
+    def remove_item(self, user_id: int, item_id: int) -> None:
+        sql = item_data.delete(
+            and_(item_data.c.user == user_id, item_data.c.item_id == item_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to remove item! profile: {user_id}, item_id: {item_id}"
+            )
+        return None

titles/sao/schema/static.py

@@ -96,6 +96,20 @@ support = Table(
     mysql_charset="utf8mb4",
 )
 
+rare_drop = Table(
+    "sao_static_rare_drop_list",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("questRareDropId", Integer),
+    Column("commonRewardId", Integer),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "questRareDropId", "commonRewardId", name="sao_static_rare_drop_list_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
 title = Table(
     "sao_static_title_list",
     metadata,
@@ -215,6 +229,23 @@ class SaoStaticData(BaseData):
         if result is None:
             return None
         return result.lastrowid
+
+    def put_rare_drop( self, version: int, questRareDropId: int, commonRewardId: int, enabled: bool ) -> Optional[int]:
+        sql = insert(rare_drop).values(
+            version=version,
+            questRareDropId=questRareDropId,
+            commonRewardId=commonRewardId,
+            enabled=enabled,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            questRareDropId=questRareDropId, commonRewardId=commonRewardId, version=version
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
     
     def put_title( self, version: int, titleId: int, displayName: str, requirement: int, rank: int, imageFilePath: str, enabled: bool ) -> Optional[int]:
         sql = insert(title).values(
@@ -236,6 +267,14 @@ class SaoStaticData(BaseData):
             return None
         return result.lastrowid
 
+    def get_quests_id(self, sortNo: int) -> Optional[Dict]:
+        sql = quest.select(quest.c.sortNo == sortNo)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
     def get_quests_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
         sql = quest.select(quest.c.version == version and quest.c.enabled == enabled).order_by(
             quest.c.questSceneId.asc()
@@ -246,6 +285,14 @@ class SaoStaticData(BaseData):
             return None
         return [list[2] for list in result.fetchall()]
     
+    def get_hero_id(self, heroLogId: int) -> Optional[Dict]:
+        sql = hero.select(hero.c.heroLogId == heroLogId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
     def get_hero_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
         sql = hero.select(hero.c.version == version and hero.c.enabled == enabled).order_by(
             hero.c.heroLogId.asc()
@@ -256,6 +303,14 @@ class SaoStaticData(BaseData):
             return None
         return [list[2] for list in result.fetchall()]
     
+    def get_equipment_id(self, equipmentId: int) -> Optional[Dict]:
+        sql = equipment.select(equipment.c.equipmentId == equipmentId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
     def get_equipment_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
         sql = equipment.select(equipment.c.version == version and equipment.c.enabled == enabled).order_by(
             equipment.c.equipmentId.asc()
@@ -265,6 +320,22 @@ class SaoStaticData(BaseData):
         if result is None:
             return None
         return [list[2] for list in result.fetchall()]
+
+    def get_item_id(self, itemId: int) -> Optional[Dict]:
+        sql = item.select(item.c.itemId == itemId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_rare_drop_id(self, questRareDropId: int) -> Optional[Dict]:
+        sql = rare_drop.select(rare_drop.c.questRareDropId == questRareDropId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
     
     def get_item_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
         sql = item.select(item.c.version == version and item.c.enabled == enabled).order_by(

titles/wacca/base.py

@@ -1074,17 +1074,17 @@ class WaccaBase:
                         old_score = self.data.score.get_best_score(
                             user_id, item.itemId, item.quantity
                         )
-                    if not old_score:
-                        self.data.score.put_best_score(
-                            user_id,
-                            item.itemId,
-                            item.quantity,
-                            0,
-                            [0] * 5,
-                            [0] * 13,
-                            0,
-                            0,
-                        )
+                        if not old_score:
+                            self.data.score.put_best_score(
+                                user_id,
+                                item.itemId,
+                                item.quantity,
+                                0,
+                                [0] * 5,
+                                [0] * 13,
+                                0,
+                                0,
+                            )
 
                     if item.quantity == 0:
                         item.quantity = WaccaConstants.Difficulty.HARD.value

titles/wacca/handlers/advertise.py

@@ -8,12 +8,12 @@ from titles.wacca.handlers.helpers import Notice
 class GetNewsResponseV1(BaseResponse):
     def __init__(self) -> None:
         super().__init__()
-        self.notices: list[Notice] = []
-        self.copywrightListings: list[str] = []
-        self.stoppedSongs: list[int] = []
-        self.stoppedJackets: list[int] = []
-        self.stoppedMovies: list[int] = []
-        self.stoppedIcons: list[int] = []
+        self.notices: List[Notice] = []
+        self.copywrightListings: List[str] = []
+        self.stoppedSongs: List[int] = []
+        self.stoppedJackets: List[int] = []
+        self.stoppedMovies: List[int] = []
+        self.stoppedIcons: List[int] = []
 
     def make(self) -> Dict:
         note = []
@@ -34,7 +34,7 @@ class GetNewsResponseV1(BaseResponse):
 
 
 class GetNewsResponseV2(GetNewsResponseV1):
-    stoppedProducts: list[int] = []
+    stoppedProducts: List[int] = []
 
     def make(self) -> Dict:
         super().make()
@@ -44,8 +44,8 @@ class GetNewsResponseV2(GetNewsResponseV1):
 
 
 class GetNewsResponseV3(GetNewsResponseV2):
-    stoppedNavs: list[int] = []
-    stoppedNavVoices: list[int] = []
+    stoppedNavs: List[int] = []
+    stoppedNavVoices: List[int] = []
 
     def make(self) -> Dict:
         super().make()