Merge branch 'develop' of https://gitea.tendokyu.moe/Hay1tsme/artemis into develop

core/allnet.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Any, Optional, Tuple
+from typing import Dict, List, Any, Optional, Tuple, Union
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 from twisted.web.http import Request
@@ -11,6 +11,7 @@ from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
 from time import strptime
 from os import path
+import urllib.parse
 
 from core.config import CoreConfig
 from core.utils import Utils
@@ -79,7 +80,7 @@ class AllnetServlet:
             req = AllnetPowerOnRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
-            if not req.game_id or not req.ver or not req.serial or not req.ip:
+            if not req.game_id or not req.ver or not req.serial or not req.ip or not req.firm_ver or not req.boot_ver:
                 raise AllnetRequestException(
                     f"Bad auth request params from {request_ip} - {vars(req)}"
                 )
@@ -89,10 +90,12 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        if req.format_ver == "3":
+        if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
-        else:
+        elif req.format_ver == 2:
             resp = AllnetPowerOnResponse2()
+        else:
+            resp = AllnetPowerOnResponse()
 
         self.logger.debug(f"Allnet request: {vars(req)}")        
         if req.game_id not in self.uri_registry:
@@ -103,8 +106,9 @@ class AllnetServlet:
                 )
                 self.logger.warn(msg)
 
-                resp.stat = 0
+                resp.stat = -1
-                return self.dict_to_http_form_string([vars(resp)])
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
             else:
                 self.logger.info(
@@ -113,8 +117,11 @@ class AllnetServlet:
                 resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
                 resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
                 
-                self.logger.debug(f"Allnet response: {vars(resp)}")
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                return self.dict_to_http_form_string([vars(resp)])
+                resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+                
+                self.logger.debug(f"Allnet response: {resp_str}")
+                return (resp_str + "\n").encode("utf-8")
 
         resp.uri, resp.host = self.uri_registry[req.game_id]
 
@@ -126,8 +133,9 @@ class AllnetServlet:
             )
             self.logger.warn(msg)
 
-            resp.stat = 0
+            resp.stat = -2
-            return self.dict_to_http_form_string([vars(resp)])
+            resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
         if machine is not None:
             arcade = self.data.arcade.get_arcade(machine["arcade"])
@@ -169,9 +177,13 @@ class AllnetServlet:
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
         self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
-        self.logger.debug(f"Allnet response: {vars(resp)}")
 
-        return self.dict_to_http_form_string([vars(resp)]).encode("utf-8")
+        resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+        self.logger.debug(f"Allnet response: {resp_dict}")        
+        resp_str += "\n"
+
+        return resp_str.encode("utf-8")
 
     def handle_dlorder(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
@@ -196,13 +208,13 @@ class AllnetServlet:
         self.logger.info(
             f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
         )
-        resp = AllnetDownloadOrderResponse()
+        resp = AllnetDownloadOrderResponse(serial=req.serial)
 
         if (
             not self.config.allnet.allow_online_updates
             or not self.config.allnet.update_cfg_folder
         ):
-            return self.dict_to_http_form_string([vars(resp)])
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
 
         else:  # TODO: Keychip check
             if path.exists(
@@ -217,7 +229,8 @@ class AllnetServlet:
 
             self.logger.debug(f"Sending download uri {resp.uri}")
             self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
-            return self.dict_to_http_form_string([vars(resp)])
+
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
 
     def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
         if "file" not in match:
@@ -241,6 +254,27 @@ class AllnetServlet:
         )
         return b""
 
+    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
+        req_data = request.content.getvalue()
+        sections = req_data.decode("utf-8").split("\r\n")
+        
+        req_dict = dict(urllib.parse.parse_qsl(sections[0]))
+
+        serial: Union[str, None] = req_dict.get("serial", None)
+        num_files_to_dl: Union[str, None] = req_dict.get("nb_ftd", None)
+        num_files_dld: Union[str, None] = req_dict.get("nb_dld", None)
+        dl_state: Union[str, None] = req_dict.get("dld_st", None)
+        ip = Utils.get_ip_addr(request)
+
+        if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
+            return "NG".encode()
+
+        self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
+        return "OK".encode()
+    
+    def handle_alive(self, request: Request, match: Dict) -> bytes:
+        return "OK".encode()
+
     def handle_billing_request(self, request: Request, _: Dict):
         req_dict = self.billing_req_to_dict(request.content.getvalue())
         request_ip = Utils.get_ip_addr(request)
@@ -304,7 +338,7 @@ class AllnetServlet:
 
         resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
 
-        resp_str = self.dict_to_http_form_string([vars(resp)], True)
+        resp_str = self.dict_to_http_form_string([vars(resp)])
         if resp_str is None:
             self.logger.error(f"Failed to parse response {vars(resp)}")
 
@@ -315,21 +349,6 @@ class AllnetServlet:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
         return b"naomi ok"
 
-    def kvp_to_dict(self, kvp: List[str]) -> List[Dict[str, Any]]:
-        ret: List[Dict[str, Any]] = []
-        for x in kvp:
-            items = x.split("&")
-            tmp = {}
-
-            for item in items:
-                kvp = item.split("=")
-                if len(kvp) == 2:
-                    tmp[kvp[0]] = kvp[1]
-
-            ret.append(tmp)
-
-        return ret
-
     def billing_req_to_dict(self, data: bytes):
         """
         Parses an billing request string into a python dictionary
@@ -339,7 +358,10 @@ class AllnetServlet:
             unzipped = decomp.decompress(data)
             sections = unzipped.decode("ascii").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
@@ -354,7 +376,10 @@ class AllnetServlet:
             unzipped = zlib.decompress(zipped)
             sections = unzipped.decode("utf-8").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"allnet_req_to_dict: {e} while parsing {data}")
@@ -363,7 +388,7 @@ class AllnetServlet:
     def dict_to_http_form_string(
         self,
         data: List[Dict[str, Any]],
-        crlf: bool = False,
+        crlf: bool = True,
         trailing_newline: bool = True,
     ) -> Optional[str]:
         """
@@ -373,21 +398,19 @@ class AllnetServlet:
             urlencode = ""
             for item in data:
                 for k, v in item.items():
+                    if k is None or v is None:
+                        continue
                     urlencode += f"{k}={v}&"
-
                 if crlf:
                     urlencode = urlencode[:-1] + "\r\n"
                 else:
                     urlencode = urlencode[:-1] + "\n"
-
             if not trailing_newline:
                 if crlf:
                     urlencode = urlencode[:-2]
                 else:
                     urlencode = urlencode[:-1]
-
             return urlencode
-
         except Exception as e:
             self.logger.error(f"dict_to_http_form_string: {e} while parsing {data}")
             return None
@@ -397,20 +420,19 @@ class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
         if req is None:
             raise AllnetRequestException("Request processing failed")
-        self.game_id: str = req.get("game_id", "")
+        self.game_id: str = req.get("game_id", None)
-        self.ver: str = req.get("ver", "")
+        self.ver: str = req.get("ver", None)
-        self.serial: str = req.get("serial", "")
+        self.serial: str = req.get("serial", None)
-        self.ip: str = req.get("ip", "")
+        self.ip: str = req.get("ip", None)
-        self.firm_ver: str = req.get("firm_ver", "")
+        self.firm_ver: str = req.get("firm_ver", None)
-        self.boot_ver: str = req.get("boot_ver", "")
+        self.boot_ver: str = req.get("boot_ver", None)
-        self.encode: str = req.get("encode", "")
+        self.encode: str = req.get("encode", "EUC-JP")
-        self.hops = int(req.get("hops", "0"))
+        self.hops = int(req.get("hops", "-1"))
-        self.format_ver = req.get("format_ver", "2")
+        self.format_ver = float(req.get("format_ver", "1.00"))
-        self.token = int(req.get("token", "0"))
+        self.token: str = req.get("token", "0")
 
-
+class AllnetPowerOnResponse:
-class AllnetPowerOnResponse3:
+    def __init__(self) -> None:
-    def __init__(self, token) -> None:
         self.stat = 1
         self.uri = ""
         self.host = ""
@@ -422,39 +444,44 @@ class AllnetPowerOnResponse3:
         self.region_name1 = ""
         self.region_name2 = ""
         self.region_name3 = ""        
-        self.country = "JPN"
-        self.allnet_id = "123"
-        self.client_timezone = "+0900"
-        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
-            "%Y-%m-%dT%H:%M:%SZ"
-        )
         self.setting = "1"
-        self.res_ver = "3"
-        self.token = str(token)
-
-
-class AllnetPowerOnResponse2:
-    def __init__(self) -> None:
-        self.stat = 1
-        self.uri = ""
-        self.host = ""
-        self.place_id = "123"
-        self.name = "ARTEMiS"
-        self.nickname = "ARTEMiS"
-        self.region0 = "1"
-        self.region_name0 = "W"
-        self.region_name1 = "X"
-        self.region_name2 = "Y"
-        self.region_name3 = "Z"
-        self.country = "JPN"
         self.year = datetime.now().year
         self.month = datetime.now().month
         self.day = datetime.now().day
         self.hour = datetime.now().hour
         self.minute = datetime.now().minute
         self.second = datetime.now().second
-        self.setting = "1"
+
-        self.timezone = "+0900"
+class AllnetPowerOnResponse3(AllnetPowerOnResponse):
+    def __init__(self, token) -> None:
+        super().__init__()
+
+        # Added in v3
+        self.country = "JPN"
+        self.allnet_id = "123"
+        self.client_timezone = "+0900"
+        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
+            "%Y-%m-%dT%H:%M:%SZ"
+        )
+        self.res_ver = "3"
+        self.token = token
+
+        # Removed in v3
+        self.year = None
+        self.month = None
+        self.day = None
+        self.hour = None
+        self.minute = None
+        self.second = None
+
+
+class AllnetPowerOnResponse2(AllnetPowerOnResponse):
+    def __init__(self) -> None:
+        super().__init__()
+
+        # Added in v2
+        self.country = "JPN"
+        self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
 

core/data/schema/versions/SDEZ_5_rollback.sql

@@ -0,0 +1,78 @@
+DELETE FROM mai2_static_event WHERE version < 13;
+UPDATE mai2_static_event SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_music WHERE version < 13;
+UPDATE mai2_static_music SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_ticket WHERE version < 13;
+UPDATE mai2_static_ticket SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_cards WHERE version < 13;
+UPDATE mai2_static_cards SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_detail WHERE version < 13;
+UPDATE mai2_profile_detail SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_extend WHERE version < 13;
+UPDATE mai2_profile_extend SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_option WHERE version < 13;
+UPDATE mai2_profile_option SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_ghost WHERE version < 13;
+UPDATE mai2_profile_ghost SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_rating WHERE version < 13;
+UPDATE mai2_profile_rating SET version = version - 13 WHERE version >= 13;
+
+DROP TABLE maimai_score_best;
+DROP TABLE maimai_playlog;
+DROP TABLE maimai_profile_detail;
+DROP TABLE maimai_profile_option;
+DROP TABLE maimai_profile_web_option;
+DROP TABLE maimai_profile_grade_status;
+
+ALTER TABLE mai2_item_character DROP COLUMN point;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NOT NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NOT NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NOT NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NOT NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NOT NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rank int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NOT NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;

core/data/schema/versions/SDEZ_6_rollback.sql

@@ -0,0 +1 @@
+DROP TABLE aime.mai2_profile_consec_logins;

core/data/schema/versions/SDEZ_6_upgrade.sql

@@ -0,0 +1,62 @@
+UPDATE mai2_static_event SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_music SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_ticket SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_cards SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_detail  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_extend  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_option  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_ghost  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_rating  SET version = version + 13 WHERE version < 1000;
+
+ALTER TABLE mai2_item_character ADD point int(11) NULL;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rank int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;

core/data/schema/versions/SDEZ_7_upgrade.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `mai2_profile_consec_logins` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `logins` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `mai2_profile_consec_logins_uk` (`user`,`version`),
+  CONSTRAINT `mai2_profile_consec_logins_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

docs/game_specific_info.md

@@ -127,28 +127,50 @@ Config file is located in `config/cxb.yaml`.
 
 ### SDEZ
 
-| Version ID | Version Name            |
+| Game Code | Version ID | Version Name            |
-|------------|-------------------------|
+|-----------|------------|-------------------------|
-| 0          | maimai DX               |
+
-| 1          | maimai DX PLUS          |
+
-| 2          | maimai DX Splash        |
+For versions pre-dx
-| 3          | maimai DX Splash PLUS   |
+| Game Code | Version ID | Version Name            |
-| 4          | maimai DX UNiVERSE      |
+|-----------|------------|-------------------------|
-| 5          | maimai DX UNiVERSE PLUS |
+| SBXL      | 0          | maimai                  |
-| 6          | maimai DX FESTiVAL      |
+| SBXL      | 1          | maimai PLUS             |
+| SBZF      | 2          | maimai GreeN            |
+| SBZF      | 3          | maimai GreeN PLUS       |
+| SDBM      | 4          | maimai ORANGE           |
+| SDBM      | 5          | maimai ORANGE PLUS      |
+| SDCQ      | 6          | maimai PiNK             |
+| SDCQ      | 7          | maimai PiNK PLUS        |
+| SDDK      | 8          | maimai MURASAKI         |
+| SDDK      | 9          | maimai MURASAKI PLUS    |
+| SDDZ      | 10         | maimai MILK             |
+| SDDZ      | 11         | maimai MILK PLUS        |
+| SDEY      | 12         | maimai FiNALE           |
+| SDEZ      | 13         | maimai DX               |
+| SDEZ      | 14         | maimai DX PLUS          |
+| SDEZ      | 15         | maimai DX Splash        |
+| SDEZ      | 16         | maimai DX Splash PLUS   |
+| SDEZ      | 17         | maimai DX Universe      |
+| SDEZ      | 18         | maimai DX Universe PLUS |
+| SDEZ      | 19         | maimai DX Festival      |
 
 ### Importer
 
 In order to use the importer locate your game installation folder and execute:
-
+DX:
 ```shell
-python read.py --series SDEZ --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --series <Game Code> --version <Version ID> --binfolder /path/to/StreamingAssets --optfolder /path/to/game/option/folder
+```
+Pre-DX:
+```shell
+python read.py --series <Game Code> --version <Version ID> --binfolder /path/to/data --optfolder /path/to/patch/data
 ```
-
 The importer for maimai DX will import Events, Music and Tickets.
 
-**NOTE: It is required to use the importer because the game will
+The importer for maimai Pre-DX will import Events and Music. Not all games will have patch data. Milk - Finale have file encryption, and need an AES key. That key is not provided by the developers. For games that do use encryption, provide the key, as a hex string, with the `--extra` flag. Ex `--extra 00112233445566778899AABBCCDDEEFF`
-crash without Events!**
+
+**Important: It is required to use the importer because some games may not function properly or even crash without Events!**
 
 ### Database upgrade
 
@@ -157,6 +179,7 @@ Always make sure your database (tables) are up-to-date, to do so go to the `core
 ```shell
 python dbutils.py --game SDEZ upgrade
 ```
+Pre-Dx uses the same database as DX, so only upgrade using the SDEZ game code!
 
 ## Hatsune Miku Project Diva
 
@@ -398,6 +421,41 @@ Always make sure your database (tables) are up-to-date, to do so go to the `core
 python dbutils.py --game SDFE upgrade
 ```
 
+### VIP Rewards
+Below is a list of VIP rewards. Currently, VIP is not implemented, and thus these are not obtainable. These 23 rewards were distributed once per month for VIP users on the real network.
+
+	Plates:
+		211004 リッチ
+		211018 特盛えりざべす
+		211025 イースター
+		211026 特盛りりぃ
+		311004 ファンシー
+		311005 インカンテーション
+		311014 夜明け
+		311015 ネイビー
+		311016 特盛るーん
+	
+	Ring Colors:
+		203002 Gold Rushイエロー
+		203009 トロピカル
+		303005 ネイチャー
+	
+	Icons:
+		202020 どらみんぐ
+		202063 ユニコーン
+		202086 ゴリラ
+		302014 ローズ
+		302015 ファラオ
+		302045 肉球
+		302046 WACCA
+		302047 WACCA Lily
+		302048 WACCA Reverse
+	
+	Note Sound Effect:
+		205002 テニス
+		205008 シャワー
+		305003 タンバリンMk-Ⅱ
+
 ## SAO
 
 ### SDEW

example_config/mai2.yaml

@@ -1,3 +1,8 @@
 server:
   enable: True
   loglevel: "info"
+
+deliver:
+  enable: False
+  udbdl_enable: False
+  content_folder: ""

index.py

@@ -63,6 +63,27 @@ class HttpDispatcher(resource.Resource):
             action="handle_dlorder",
             conditions=dict(method=["POST"]),
         )
+        self.map_post.connect(
+            "allnet_loaderstaterecorder",
+            "/sys/servlet/LoaderStateRecorder",
+            controller="allnet",
+            action="handle_loaderstaterecorder",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_get.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["GET"]),
+        )
         self.map_post.connect(
             "allnet_billing",
             "/request",
@@ -111,7 +132,6 @@ class HttpDispatcher(resource.Resource):
         )
 
     def render_GET(self, request: Request) -> bytes:
-        self.logger.debug(request.uri)
         test = self.map_get.match(request.uri.decode())
         client_ip = Utils.get_ip_addr(request)
 
@@ -161,9 +181,16 @@ class HttpDispatcher(resource.Resource):
 
         if type(ret) == str:
             return ret.encode()
-        elif type(ret) == bytes:
+        
+        elif type(ret) == bytes or type(ret) == tuple: # allow for bytes or tuple (data, response code) responses
             return ret
+        
+        elif ret is None:
+            self.logger.warn(f"None returned by controller for {request.uri.decode()} endpoint")
+            return b""
+        
         else:
+            self.logger.warn(f"Unknown data type returned by controller for {request.uri.decode()} endpoint")
             return b""
 
 

titles/mai2/__init__.py

@@ -6,5 +6,14 @@ from titles.mai2.read import Mai2Reader
 index = Mai2Servlet
 database = Mai2Data
 reader = Mai2Reader
-game_codes = [Mai2Constants.GAME_CODE]
+game_codes = [
-current_schema_version = 5
+    Mai2Constants.GAME_CODE_DX,
+    Mai2Constants.GAME_CODE_FINALE,
+    Mai2Constants.GAME_CODE_MILK,
+    Mai2Constants.GAME_CODE_MURASAKI,
+    Mai2Constants.GAME_CODE_PINK,
+    Mai2Constants.GAME_CODE_ORANGE,
+    Mai2Constants.GAME_CODE_GREEN,
+    Mai2Constants.GAME_CODE,
+]
+current_schema_version = 7

titles/mai2/base.py

@@ -12,40 +12,35 @@ class Mai2Base:
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         self.core_config = cfg
         self.game_config = game_cfg
-        self.game = Mai2Constants.GAME_CODE
+        self.version = Mai2Constants.VER_MAIMAI
-        self.version = Mai2Constants.VER_MAIMAI_DX
         self.data = Mai2Data(cfg)
         self.logger = logging.getLogger("mai2")
+        self.can_deliver = False
+        self.can_usbdl = False
+        self.old_server = ""
         
         if self.core_config.server.is_develop and self.core_config.title.port > 0:
-            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEZ/100/"
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/197/"
         
         else:
-            self.old_server = f"http://{self.core_config.title.hostname}/SDEZ/100/"
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/197/"
 
     def handle_get_game_setting_api_request(self, data: Dict):
-        # TODO: See if making this epoch 0 breaks things
-        reboot_start = date.strftime(
-            datetime.now() + timedelta(hours=3), Mai2Constants.DATE_TIME_FORMAT
-        )
-        reboot_end = date.strftime(
-            datetime.now() + timedelta(hours=4), Mai2Constants.DATE_TIME_FORMAT
-        )
         return {            
+            "isDevelop": False,
+            "isAouAccession": False,
             "gameSetting": {
-                "isMaintenance": "false",
+                "isMaintenance": False,
-                "requestInterval": 10,
+                "requestInterval": 1800,
-                "rebootStartTime": reboot_start,
+                "rebootStartTime": "2020-01-01 07:00:00.0",
-                "rebootEndTime": reboot_end,
+                "rebootEndTime": "2020-01-01 07:59:59.0",
-                "movieUploadLimit": 10000,
+                "movieUploadLimit": 100,
-                "movieStatus": 0,
+                "movieStatus": 1,
-                "movieServerUri": "",
+                "movieServerUri": self.old_server + "api/movie" if self.game_config.uploads.movies else "movie",
-                "deliverServerUri": "",
+                "deliverServerUri": self.old_server + "deliver/" if self.can_deliver and self.game_config.deliver.enable else "",
-                "oldServerUri": self.old_server,
+                "oldServerUri": self.old_server + "old",
-                "usbDlServerUri": "",
+                "usbDlServerUri": self.old_server + "usbdl/" if self.can_deliver and self.game_config.deliver.udbdl_enable else "",
-                "rebootInterval": 0,
             },
-            "isAouAccession": "true",
         }
 
     def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
@@ -58,7 +53,7 @@ class Mai2Base:
     def handle_get_game_event_api_request(self, data: Dict) -> Dict:
         events = self.data.static.get_enabled_events(self.version)
         events_lst = []
-        if events is None:
+        if events is None or not events:
             self.logger.warn("No enabled events, did you run the reader?")
             return {"type": data["type"], "length": 0, "gameEventList": []}
 
@@ -117,43 +112,35 @@ class Mai2Base:
         return {"returnCode": 1, "apiName": "UpsertClientTestmodeApi"}
 
     def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+        p = self.data.profile.get_profile_detail(data["userId"], self.version, False)
-        o = self.data.profile.get_profile_option(data["userId"], self.version)
+        w = self.data.profile.get_web_option(data["userId"], self.version)
-        if p is None or o is None:
+        if p is None or w is None:
             return {}  # Register
         profile = p._asdict()
-        option = o._asdict()
+        web_opt = w._asdict()
 
         return {
             "userId": data["userId"],
             "userName": profile["userName"],
             "isLogin": False,
-            "lastGameId": profile["lastGameId"],
             "lastDataVersion": profile["lastDataVersion"],
-            "lastRomVersion": profile["lastRomVersion"],
+            "lastLoginDate": profile["lastPlayDate"],
-            "lastLoginDate": profile["lastLoginDate"],
             "lastPlayDate": profile["lastPlayDate"],
             "playerRating": profile["playerRating"],
-            "nameplateId": 0,  # Unused
+            "nameplateId": profile["nameplateId"],
-            "iconId": profile["iconId"],
-            "trophyId": 0,  # Unused
-            "partnerId": profile["partnerId"],
             "frameId": profile["frameId"],
-            "dispRate": option[
+            "iconId": profile["iconId"],
-                "dispRate"
+            "trophyId": profile["trophyId"],
-            ],  # 0: all/begin, 1: disprate, 2: dispDan, 3: hide, 4: end
+            "dispRate": web_opt["dispRate"],  # 0: all, 1: dispRate, 2: dispDan, 3: hide
-            "totalAwake": profile["totalAwake"],
+            "dispRank": web_opt["dispRank"],
-            "isNetMember": profile["isNetMember"],
+            "dispHomeRanker": web_opt["dispHomeRanker"],
-            "dailyBonusDate": profile["dailyBonusDate"],
+            "dispTotalLv": web_opt["dispTotalLv"],
-            "headPhoneVolume": option["headPhoneVolume"],
+            "totalLv": profile["totalLv"],
-            "isInherit": False,  # Not sure what this is or does??
-            "banState": profile["banState"]
-            if profile["banState"] is not None
-            else 0,  # New with uni+
         }
 
     def handle_user_login_api_request(self, data: Dict) -> Dict:
         profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        consec = self.data.profile.get_consec_login(data["userId"], self.version)
 
         if profile is not None:
             lastLoginDate = profile["lastLoginDate"]
@@ -165,12 +152,31 @@ class Mai2Base:
             loginCt = 0
             lastLoginDate = "2017-12-05 07:00:00.0"
         
+        if consec is None or not consec:
+            consec_ct = 1
+        
+        else:
+            lastlogindate_ = datetime.strptime(profile["lastLoginDate"], "%Y-%m-%d %H:%M:%S.%f").timestamp()
+            today_midnight = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp()
+            yesterday_midnight = today_midnight - 86400
+
+            if lastlogindate_ < today_midnight:
+                consec_ct = consec['logins'] + 1                
+                self.data.profile.add_consec_login(data["userId"], self.version)
+            
+            elif lastlogindate_ < yesterday_midnight:
+                consec_ct = 1
+                self.data.profile.reset_consec_login(data["userId"], self.version)
+
+            else:
+                consec_ct = consec['logins']
+        
+
         return {
             "returnCode": 1,
             "lastLoginDate": lastLoginDate,
             "loginCount": loginCt,
-            "consecutiveLoginCount": 0,  # We don't really have a way to track this...
+            "consecutiveLoginCount": consec_ct, # Number of consecutive days we've logged in.
-            "loginId": loginCt,  # Used with the playlog!
         }
 
     def handle_upload_user_playlog_api_request(self, data: Dict) -> Dict:
@@ -202,12 +208,35 @@ class Mai2Base:
         upsert = data["upsertUserAll"]
 
         if "userData" in upsert and len(upsert["userData"]) > 0:
-            upsert["userData"][0]["isNetMember"] = 1
             upsert["userData"][0].pop("accessCode")
+            upsert["userData"][0].pop("userId")
+
             self.data.profile.put_profile_detail(
-                user_id, self.version, upsert["userData"][0]
+                user_id, self.version, upsert["userData"][0], False
             )
         
+        if "userWebOption" in upsert and len(upsert["userWebOption"]) > 0:            
+            upsert["userWebOption"][0]["isNetMember"] = True
+            self.data.profile.put_web_option(
+                user_id, self.version, upsert["userWebOption"][0]
+            )
+
+        if "userGradeStatusList" in upsert and len(upsert["userGradeStatusList"]) > 0:
+            self.data.profile.put_grade_status(
+                user_id, upsert["userGradeStatusList"][0]
+            )
+
+        if "userBossList" in upsert and len(upsert["userBossList"]) > 0:
+            self.data.profile.put_boss_list(
+                user_id, upsert["userBossList"][0]
+            )
+
+        if "userPlaylogList" in upsert and len(upsert["userPlaylogList"]) > 0:
+            for playlog in upsert["userPlaylogList"]:
+                self.data.score.put_playlog(
+                    user_id, playlog, False
+                )
+
         if "userExtend" in upsert and len(upsert["userExtend"]) > 0:
             self.data.profile.put_profile_extend(
                 user_id, self.version, upsert["userExtend"][0]
@@ -215,11 +244,15 @@ class Mai2Base:
 
         if "userGhost" in upsert:
             for ghost in upsert["userGhost"]:
-                self.data.profile.put_profile_extend(user_id, self.version, ghost)
+                self.data.profile.put_profile_ghost(user_id, self.version, ghost)
+
+        if "userRecentRatingList" in upsert:
+            self.data.profile.put_recent_rating(user_id, upsert["userRecentRatingList"])
 
         if "userOption" in upsert and len(upsert["userOption"]) > 0:
+            upsert["userOption"][0].pop("userId")
             self.data.profile.put_profile_option(
-                user_id, self.version, upsert["userOption"][0]
+                user_id, self.version, upsert["userOption"][0], False
             )
 
         if "userRatingList" in upsert and len(upsert["userRatingList"]) > 0:
@@ -228,9 +261,8 @@ class Mai2Base:
             )
 
         if "userActivityList" in upsert and len(upsert["userActivityList"]) > 0:
-            for k, v in upsert["userActivityList"][0].items():
+            for act in upsert["userActivityList"]:
-                for act in v:
+                self.data.profile.put_profile_activity(user_id, act)
-                    self.data.profile.put_profile_activity(user_id, act)
 
         if "userChargeList" in upsert and len(upsert["userChargeList"]) > 0:
             for charge in upsert["userChargeList"]:
@@ -250,12 +282,9 @@ class Mai2Base:
 
         if "userCharacterList" in upsert and len(upsert["userCharacterList"]) > 0:
             for char in upsert["userCharacterList"]:
-                self.data.item.put_character(
+                self.data.item.put_character_(
                     user_id,
-                    char["characterId"],
+                    char
-                    char["level"],
-                    char["awakening"],
-                    char["useCount"],
                 )
 
         if "userItemList" in upsert and len(upsert["userItemList"]) > 0:
@@ -265,7 +294,7 @@ class Mai2Base:
                     int(item["itemKind"]),
                     item["itemId"],
                     item["stock"],
-                    item["isValid"],
+                    True
                 )
 
         if "userLoginBonusList" in upsert and len(upsert["userLoginBonusList"]) > 0:
@@ -291,7 +320,7 @@ class Mai2Base:
 
         if "userMusicDetailList" in upsert and len(upsert["userMusicDetailList"]) > 0:
             for music in upsert["userMusicDetailList"]:
-                self.data.score.put_best_score(user_id, music)
+                self.data.score.put_best_score(user_id, music, False)
 
         if "userCourseList" in upsert and len(upsert["userCourseList"]) > 0:
             for course in upsert["userCourseList"]:
@@ -319,7 +348,7 @@ class Mai2Base:
         return {"returnCode": 1}
 
     def handle_get_user_data_api_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        profile = self.data.profile.get_profile_detail(data["userId"], self.version, False)
         if profile is None:
             return
 
@@ -343,7 +372,7 @@ class Mai2Base:
         return {"userId": data["userId"], "userExtend": extend_dict}
 
     def handle_get_user_option_api_request(self, data: Dict) -> Dict:
-        options = self.data.profile.get_profile_option(data["userId"], self.version)
+        options = self.data.profile.get_profile_option(data["userId"], self.version, False)
         if options is None:
             return
 
@@ -413,6 +442,25 @@ class Mai2Base:
             "userChargeList": user_charge_list,
         }
 
+    def handle_get_user_present_api_request(self, data: Dict) -> Dict:
+        return { "userId": data.get("userId", 0), "length": 0, "userPresentList": []}
+    
+    def handle_get_transfer_friend_api_request(self, data: Dict) -> Dict:
+        return {}
+
+    def handle_get_user_present_event_api_request(self, data: Dict) -> Dict:
+        return { "userId": data.get("userId", 0), "length": 0, "userPresentEventList": []}
+    
+    def handle_get_user_boss_api_request(self, data: Dict) -> Dict:
+        b = self.data.profile.get_boss_list(data["userId"])
+        if b is None:
+            return { "userId": data.get("userId", 0), "userBossData": {}}
+        boss_lst = b._asdict()
+        boss_lst.pop("id")
+        boss_lst.pop("user")
+
+        return { "userId": data.get("userId", 0), "userBossData": boss_lst}
+
     def handle_get_user_item_api_request(self, data: Dict) -> Dict:
         kind = int(data["nextIndex"] / 10000000000)
         next_idx = int(data["nextIndex"] % 10000000000)
@@ -449,6 +497,8 @@ class Mai2Base:
             tmp = chara._asdict()
             tmp.pop("id")
             tmp.pop("user")
+            tmp.pop("awakening")
+            tmp.pop("useCount")
             chara_list.append(tmp)
 
         return {"userId": data["userId"], "userCharacterList": chara_list}
@@ -482,6 +532,16 @@ class Mai2Base:
 
         return {"userId": data["userId"], "userGhost": ghost_dict}
 
+    def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
+        rating = self.data.profile.get_recent_rating(data["userId"])
+        if rating is None:
+            return
+        
+        r = rating._asdict()
+        lst = r.get("userRecentRatingList", [])
+        
+        return {"userId": data["userId"], "length": len(lst), "userRecentRatingList": lst}
+
     def handle_get_user_rating_api_request(self, data: Dict) -> Dict:
         rating = self.data.profile.get_profile_rating(data["userId"], self.version)
         if rating is None:
@@ -645,6 +705,31 @@ class Mai2Base:
     def handle_get_user_region_api_request(self, data: Dict) -> Dict:
         return {"userId": data["userId"], "length": 0, "userRegionList": []}
     
+    def handle_get_user_web_option_api_request(self, data: Dict) -> Dict:
+        w = self.data.profile.get_web_option(data["userId"], self.version)
+        if  w is None:
+            return {"userId": data["userId"], "userWebOption": {}}
+        
+        web_opt = w._asdict()        
+        web_opt.pop("id")
+        web_opt.pop("user")
+        web_opt.pop("version")
+
+        return {"userId": data["userId"], "userWebOption": web_opt}
+
+    def handle_get_user_survival_api_request(self, data: Dict) -> Dict:
+        return {"userId": data["userId"], "length": 0, "userSurvivalList": []}
+
+    def handle_get_user_grade_api_request(self, data: Dict) -> Dict:
+        g = self.data.profile.get_grade_status(data["userId"])
+        if g is None:
+            return {"userId": data["userId"], "userGradeStatus": {}, "length": 0, "userGradeList": []}
+        grade_stat = g._asdict()
+        grade_stat.pop("id")
+        grade_stat.pop("user")
+
+        return {"userId": data["userId"], "userGradeStatus": grade_stat, "length": 0, "userGradeList": []}
+
     def handle_get_user_music_api_request(self, data: Dict) -> Dict:
         user_id = data.get("userId", 0)        
         next_index = data.get("nextIndex", 0)
@@ -656,7 +741,7 @@ class Mai2Base:
             self.logger.warn("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
             return {}
         
-        songs = self.data.score.get_best_scores(user_id)
+        songs = self.data.score.get_best_scores(user_id, is_dx=False)
         if songs is None:
             self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
             return {
@@ -683,3 +768,9 @@ class Mai2Base:
             "nextIndex": next_index,
             "userMusicList": [{"userMusicDetailList": music_detail_list}],
         }
+
+    def handle_upload_user_portrait_api_request(self, data: Dict) -> Dict:
+        self.logger.debug(data)
+
+    def handle_upload_user_photo_api_request(self, data: Dict) -> Dict:
+        self.logger.debug(data)

titles/mai2/config.py

@@ -19,7 +19,59 @@ class Mai2ServerConfig:
             )
         )
 
+class Mai2DeliverConfig:
+    def __init__(self, parent: "Mai2Config") -> None:
+        self.__config = parent
+
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "deliver", "enable", default=False
+        )
+
+    @property
+    def udbdl_enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "deliver", "udbdl_enable", default=False
+        )
+    
+    @property
+    def content_folder(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "server", "content_folder", default=""
+        )
+
+class Mai2UploadsConfig:
+    def __init__(self, parent: "Mai2Config") -> None:
+        self.__config = parent
+    
+    @property
+    def photos(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "photos", default=False
+        )
+
+    @property
+    def photos_dir(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "photos_dir", default=""
+        )
+
+    @property
+    def movies(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "movies", default=False
+        )
+
+    @property
+    def movies_dir(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "movies_dir", default=""
+        )
+
 
 class Mai2Config(dict):
     def __init__(self) -> None:
         self.server = Mai2ServerConfig(self)
+        self.deliver = Mai2DeliverConfig(self)
+        self.uploads = Mai2UploadsConfig(self)

titles/mai2/const.py

@@ -20,19 +20,53 @@ class Mai2Constants:
 
     DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 
-    GAME_CODE = "SDEZ"
+    GAME_CODE = "SBXL"    
+    GAME_CODE_GREEN = "SBZF"    
+    GAME_CODE_ORANGE = "SDBM"    
+    GAME_CODE_PINK = "SDCQ"    
+    GAME_CODE_MURASAKI = "SDDK"    
+    GAME_CODE_MILK = "SDDZ"    
+    GAME_CODE_FINALE = "SDEY"    
+    GAME_CODE_DX = "SDEZ"
 
     CONFIG_NAME = "mai2.yaml"
 
-    VER_MAIMAI_DX = 0
+    VER_MAIMAI = 0
-    VER_MAIMAI_DX_PLUS = 1
+    VER_MAIMAI_PLUS = 1
-    VER_MAIMAI_DX_SPLASH = 2
+    VER_MAIMAI_GREEN = 2
-    VER_MAIMAI_DX_SPLASH_PLUS = 3
+    VER_MAIMAI_GREEN_PLUS = 3
-    VER_MAIMAI_DX_UNIVERSE = 4
+    VER_MAIMAI_ORANGE = 4
-    VER_MAIMAI_DX_UNIVERSE_PLUS = 5
+    VER_MAIMAI_ORANGE_PLUS = 5
-    VER_MAIMAI_DX_FESTIVAL = 6
+    VER_MAIMAI_PINK = 6
+    VER_MAIMAI_PINK_PLUS = 7
+    VER_MAIMAI_MURASAKI = 8
+    VER_MAIMAI_MURASAKI_PLUS = 9
+    VER_MAIMAI_MILK = 10
+    VER_MAIMAI_MILK_PLUS = 11
+    VER_MAIMAI_FINALE = 12
+
+    VER_MAIMAI_DX = 13
+    VER_MAIMAI_DX_PLUS = 14
+    VER_MAIMAI_DX_SPLASH = 15
+    VER_MAIMAI_DX_SPLASH_PLUS = 16
+    VER_MAIMAI_DX_UNIVERSE = 17
+    VER_MAIMAI_DX_UNIVERSE_PLUS = 18
+    VER_MAIMAI_DX_FESTIVAL = 19
 
     VERSION_STRING = (
+        "maimai",
+        "maimai PLUS",
+        "maimai GreeN",
+        "maimai GreeN PLUS",
+        "maimai ORANGE",
+        "maimai ORANGE PLUS",
+        "maimai PiNK",
+        "maimai PiNK PLUS",
+        "maimai MURASAKi",
+        "maimai MURASAKi PLUS",
+        "maimai MiLK",
+        "maimai MiLK PLUS",
+        "maimai FiNALE",        
         "maimai DX",
         "maimai DX PLUS",
         "maimai DX Splash",

titles/mai2/dx.py

@@ -0,0 +1,743 @@
+from typing import Any, List, Dict
+from datetime import datetime, timedelta
+import pytz
+import json
+from random import randint
+
+from core.config import CoreConfig
+from titles.mai2.base import Mai2Base
+from titles.mai2.config import Mai2Config
+from titles.mai2.const import Mai2Constants
+
+
+class Mai2DX(Mai2Base):
+    def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
+        super().__init__(cfg, game_cfg)
+        self.version = Mai2Constants.VER_MAIMAI_DX
+        
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEZ/100/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEZ/100/"
+    
+    def handle_get_game_setting_api_request(self, data: Dict):
+        return {
+            "gameSetting": {
+                "isMaintenance": False,
+                "requestInterval": 1800,
+                "rebootStartTime": "2020-01-01 07:00:00.0",
+                "rebootEndTime": "2020-01-01 07:59:59.0",
+                "movieUploadLimit": 100,
+                "movieStatus": 1,
+                "movieServerUri": self.old_server + "movie/",
+                "deliverServerUri": self.old_server + "deliver/" if self.can_deliver and self.game_config.deliver.enable else "",
+                "oldServerUri": self.old_server + "old",
+                "usbDlServerUri": self.old_server + "usbdl/" if self.can_deliver and self.game_config.deliver.udbdl_enable else "",
+                "rebootInterval": 0,
+            },
+            "isAouAccession": False,
+        }
+
+    def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
+        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+        o = self.data.profile.get_profile_option(data["userId"], self.version)
+        if p is None or o is None:
+            return {}  # Register
+        profile = p._asdict()
+        option = o._asdict()
+
+        return {
+            "userId": data["userId"],
+            "userName": profile["userName"],
+            "isLogin": False,
+            "lastGameId": profile["lastGameId"],
+            "lastDataVersion": profile["lastDataVersion"],
+            "lastRomVersion": profile["lastRomVersion"],
+            "lastLoginDate": profile["lastLoginDate"],
+            "lastPlayDate": profile["lastPlayDate"],
+            "playerRating": profile["playerRating"],
+            "nameplateId": 0,  # Unused
+            "iconId": profile["iconId"],
+            "trophyId": 0,  # Unused
+            "partnerId": profile["partnerId"],
+            "frameId": profile["frameId"],
+            "dispRate": option[
+                "dispRate"
+            ],  # 0: all/begin, 1: disprate, 2: dispDan, 3: hide, 4: end
+            "totalAwake": profile["totalAwake"],
+            "isNetMember": profile["isNetMember"],
+            "dailyBonusDate": profile["dailyBonusDate"],
+            "headPhoneVolume": option["headPhoneVolume"],
+            "isInherit": False,  # Not sure what this is or does??
+            "banState": profile["banState"]
+            if profile["banState"] is not None
+            else 0,  # New with uni+
+        }
+
+    def handle_upload_user_playlog_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        playlog = data["userPlaylog"]
+
+        self.data.score.put_playlog(user_id, playlog)
+
+        return {"returnCode": 1, "apiName": "UploadUserPlaylogApi"}
+
+    def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        charge = data["userCharge"]
+
+        # remove the ".0" from the date string, festival only?
+        charge["purchaseDate"] = charge["purchaseDate"].replace(".0", "")
+        self.data.item.put_charge(
+            user_id,
+            charge["chargeId"],
+            charge["stock"],
+            datetime.strptime(charge["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT),
+            datetime.strptime(charge["validDate"], Mai2Constants.DATE_TIME_FORMAT),
+        )
+
+        return {"returnCode": 1, "apiName": "UpsertUserChargelogApi"}
+
+    def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        upsert = data["upsertUserAll"]
+
+        if "userData" in upsert and len(upsert["userData"]) > 0:
+            upsert["userData"][0]["isNetMember"] = 1
+            upsert["userData"][0].pop("accessCode")
+            self.data.profile.put_profile_detail(
+                user_id, self.version, upsert["userData"][0]
+            )
+
+        if "userExtend" in upsert and len(upsert["userExtend"]) > 0:
+            self.data.profile.put_profile_extend(
+                user_id, self.version, upsert["userExtend"][0]
+            )
+
+        if "userGhost" in upsert:
+            for ghost in upsert["userGhost"]:
+                self.data.profile.put_profile_extend(user_id, self.version, ghost)
+
+        if "userOption" in upsert and len(upsert["userOption"]) > 0:
+            self.data.profile.put_profile_option(
+                user_id, self.version, upsert["userOption"][0]
+            )
+
+        if "userRatingList" in upsert and len(upsert["userRatingList"]) > 0:
+            self.data.profile.put_profile_rating(
+                user_id, self.version, upsert["userRatingList"][0]
+            )
+
+        if "userActivityList" in upsert and len(upsert["userActivityList"]) > 0:
+            for k, v in upsert["userActivityList"][0].items():
+                for act in v:
+                    self.data.profile.put_profile_activity(user_id, act)
+
+        if "userChargeList" in upsert and len(upsert["userChargeList"]) > 0:
+            for charge in upsert["userChargeList"]:
+                # remove the ".0" from the date string, festival only?
+                charge["purchaseDate"] = charge["purchaseDate"].replace(".0", "")
+                self.data.item.put_charge(
+                    user_id,
+                    charge["chargeId"],
+                    charge["stock"],
+                    datetime.strptime(
+                        charge["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT
+                    ),
+                    datetime.strptime(
+                        charge["validDate"], Mai2Constants.DATE_TIME_FORMAT
+                    ),
+                )
+
+        if "userCharacterList" in upsert and len(upsert["userCharacterList"]) > 0:
+            for char in upsert["userCharacterList"]:
+                self.data.item.put_character(
+                    user_id,
+                    char["characterId"],
+                    char["level"],
+                    char["awakening"],
+                    char["useCount"],
+                )
+
+        if "userItemList" in upsert and len(upsert["userItemList"]) > 0:
+            for item in upsert["userItemList"]:
+                self.data.item.put_item(
+                    user_id,
+                    int(item["itemKind"]),
+                    item["itemId"],
+                    item["stock"],
+                    item["isValid"],
+                )
+
+        if "userLoginBonusList" in upsert and len(upsert["userLoginBonusList"]) > 0:
+            for login_bonus in upsert["userLoginBonusList"]:
+                self.data.item.put_login_bonus(
+                    user_id,
+                    login_bonus["bonusId"],
+                    login_bonus["point"],
+                    login_bonus["isCurrent"],
+                    login_bonus["isComplete"],
+                )
+
+        if "userMapList" in upsert and len(upsert["userMapList"]) > 0:
+            for map in upsert["userMapList"]:
+                self.data.item.put_map(
+                    user_id,
+                    map["mapId"],
+                    map["distance"],
+                    map["isLock"],
+                    map["isClear"],
+                    map["isComplete"],
+                )
+
+        if "userMusicDetailList" in upsert and len(upsert["userMusicDetailList"]) > 0:
+            for music in upsert["userMusicDetailList"]:
+                self.data.score.put_best_score(user_id, music)
+
+        if "userCourseList" in upsert and len(upsert["userCourseList"]) > 0:
+            for course in upsert["userCourseList"]:
+                self.data.score.put_course(user_id, course)
+
+        if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0:
+            for fav in upsert["userFavoriteList"]:
+                self.data.item.put_favorite(user_id, fav["kind"], fav["itemIdList"])
+
+        if (
+            "userFriendSeasonRankingList" in upsert
+            and len(upsert["userFriendSeasonRankingList"]) > 0
+        ):
+            for fsr in upsert["userFriendSeasonRankingList"]:
+                fsr["recordDate"] = (
+                    datetime.strptime(
+                        fsr["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+                    ),
+                )
+                self.data.item.put_friend_season_ranking(user_id, fsr)
+
+        return {"returnCode": 1, "apiName": "UpsertUserAllApi"}
+
+    def handle_user_logout_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}
+
+    def handle_get_user_data_api_request(self, data: Dict) -> Dict:
+        profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        if profile is None:
+            return
+
+        profile_dict = profile._asdict()
+        profile_dict.pop("id")
+        profile_dict.pop("user")
+        profile_dict.pop("version")
+
+        return {"userId": data["userId"], "userData": profile_dict}
+
+    def handle_get_user_extend_api_request(self, data: Dict) -> Dict:
+        extend = self.data.profile.get_profile_extend(data["userId"], self.version)
+        if extend is None:
+            return
+
+        extend_dict = extend._asdict()
+        extend_dict.pop("id")
+        extend_dict.pop("user")
+        extend_dict.pop("version")
+
+        return {"userId": data["userId"], "userExtend": extend_dict}
+
+    def handle_get_user_option_api_request(self, data: Dict) -> Dict:
+        options = self.data.profile.get_profile_option(data["userId"], self.version)
+        if options is None:
+            return
+
+        options_dict = options._asdict()
+        options_dict.pop("id")
+        options_dict.pop("user")
+        options_dict.pop("version")
+
+        return {"userId": data["userId"], "userOption": options_dict}
+
+    def handle_get_user_card_api_request(self, data: Dict) -> Dict:
+        user_cards = self.data.item.get_cards(data["userId"])
+        if user_cards is None:
+            return {"userId": data["userId"], "nextIndex": 0, "userCardList": []}
+
+        max_ct = data["maxCount"]
+        next_idx = data["nextIndex"]
+        start_idx = next_idx
+        end_idx = max_ct + start_idx
+
+        if len(user_cards[start_idx:]) > max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        card_list = []
+        for card in user_cards:
+            tmp = card._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            tmp["startDate"] = datetime.strftime(
+                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["endDate"] = datetime.strftime(
+                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            card_list.append(tmp)
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userCardList": card_list[start_idx:end_idx],
+        }
+
+    def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
+        user_charges = self.data.item.get_charges(data["userId"])
+        if user_charges is None:
+            return {"userId": data["userId"], "length": 0, "userChargeList": []}
+
+        user_charge_list = []
+        for charge in user_charges:
+            tmp = charge._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            tmp["purchaseDate"] = datetime.strftime(
+                tmp["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["validDate"] = datetime.strftime(
+                tmp["validDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+
+            user_charge_list.append(tmp)
+
+        return {
+            "userId": data["userId"],
+            "length": len(user_charge_list),
+            "userChargeList": user_charge_list,
+        }
+
+    def handle_get_user_item_api_request(self, data: Dict) -> Dict:
+        kind = int(data["nextIndex"] / 10000000000)
+        next_idx = int(data["nextIndex"] % 10000000000)
+        user_item_list = self.data.item.get_items(data["userId"], kind)
+
+        items: list[Dict[str, Any]] = []
+        for i in range(next_idx, len(user_item_list)):
+            tmp = user_item_list[i]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            items.append(tmp)
+            if len(items) >= int(data["maxCount"]):
+                break
+
+        xout = kind * 10000000000 + next_idx + len(items)
+
+        if len(items) < int(data["maxCount"]):
+            next_idx = 0
+        else:
+            next_idx = xout
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "itemKind": kind,
+            "userItemList": items,
+        }
+
+    def handle_get_user_character_api_request(self, data: Dict) -> Dict:
+        characters = self.data.item.get_characters(data["userId"])
+
+        chara_list = []
+        for chara in characters:
+            tmp = chara._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            chara_list.append(tmp)
+
+        return {"userId": data["userId"], "userCharacterList": chara_list}
+
+    def handle_get_user_favorite_api_request(self, data: Dict) -> Dict:
+        favorites = self.data.item.get_favorites(data["userId"], data["itemKind"])
+        if favorites is None:
+            return
+
+        userFavs = []
+        for fav in favorites:
+            userFavs.append(
+                {
+                    "userId": data["userId"],
+                    "itemKind": fav["itemKind"],
+                    "itemIdList": fav["itemIdList"],
+                }
+            )
+
+        return {"userId": data["userId"], "userFavoriteData": userFavs}
+
+    def handle_get_user_ghost_api_request(self, data: Dict) -> Dict:
+        ghost = self.data.profile.get_profile_ghost(data["userId"], self.version)
+        if ghost is None:
+            return
+
+        ghost_dict = ghost._asdict()
+        ghost_dict.pop("user")
+        ghost_dict.pop("id")
+        ghost_dict.pop("version_int")
+
+        return {"userId": data["userId"], "userGhost": ghost_dict}
+
+    def handle_get_user_rating_api_request(self, data: Dict) -> Dict:
+        rating = self.data.profile.get_profile_rating(data["userId"], self.version)
+        if rating is None:
+            return
+
+        rating_dict = rating._asdict()
+        rating_dict.pop("user")
+        rating_dict.pop("id")
+        rating_dict.pop("version")
+
+        return {"userId": data["userId"], "userRating": rating_dict}
+
+    def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
+        """
+        kind 1 is playlist, kind 2 is music list
+        """
+        playlist = self.data.profile.get_profile_activity(data["userId"], 1)
+        musiclist = self.data.profile.get_profile_activity(data["userId"], 2)
+        if playlist is None or musiclist is None:
+            return
+
+        plst = []
+        mlst = []
+
+        for play in playlist:
+            tmp = play._asdict()
+            tmp["id"] = tmp["activityId"]
+            tmp.pop("activityId")
+            tmp.pop("user")
+            plst.append(tmp)
+
+        for music in musiclist:
+            tmp = music._asdict()
+            tmp["id"] = tmp["activityId"]
+            tmp.pop("activityId")
+            tmp.pop("user")
+            mlst.append(tmp)
+
+        return {"userActivity": {"playList": plst, "musicList": mlst}}
+
+    def handle_get_user_course_api_request(self, data: Dict) -> Dict:
+        user_courses = self.data.score.get_courses(data["userId"])
+        if user_courses is None:
+            return {"userId": data["userId"], "nextIndex": 0, "userCourseList": []}
+
+        course_list = []
+        for course in user_courses:
+            tmp = course._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            course_list.append(tmp)
+
+        return {"userId": data["userId"], "nextIndex": 0, "userCourseList": course_list}
+
+    def handle_get_user_portrait_api_request(self, data: Dict) -> Dict:
+        # No support for custom pfps
+        return {"length": 0, "userPortraitList": []}
+
+    def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict:
+        friend_season_ranking = self.data.item.get_friend_season_ranking(data["userId"])
+        if friend_season_ranking is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userFriendSeasonRankingList": [],
+            }
+
+        friend_season_ranking_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(friend_season_ranking)):
+            tmp = friend_season_ranking[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            tmp["recordDate"] = datetime.strftime(
+                tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+            )
+            friend_season_ranking_list.append(tmp)
+
+            if len(friend_season_ranking_list) >= max_ct:
+                break
+
+        if len(friend_season_ranking) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userFriendSeasonRankingList": friend_season_ranking_list,
+        }
+
+    def handle_get_user_map_api_request(self, data: Dict) -> Dict:
+        maps = self.data.item.get_maps(data["userId"])
+        if maps is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userMapList": [],
+            }
+
+        map_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(maps)):
+            tmp = maps[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            map_list.append(tmp)
+
+            if len(map_list) >= max_ct:
+                break
+
+        if len(maps) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userMapList": map_list,
+        }
+
+    def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
+        login_bonuses = self.data.item.get_login_bonuses(data["userId"])
+        if login_bonuses is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userLoginBonusList": [],
+            }
+
+        login_bonus_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(login_bonuses)):
+            tmp = login_bonuses[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            login_bonus_list.append(tmp)
+
+            if len(login_bonus_list) >= max_ct:
+                break
+
+        if len(login_bonuses) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userLoginBonusList": login_bonus_list,
+        }
+
+    def handle_get_user_region_api_request(self, data: Dict) -> Dict:
+        return {"userId": data["userId"], "length": 0, "userRegionList": []}
+
+    def handle_get_user_music_api_request(self, data: Dict) -> Dict:
+        songs = self.data.score.get_best_scores(data["userId"])
+        music_detail_list = []
+        next_index = 0
+
+        if songs is not None:
+            for song in songs:
+                tmp = song._asdict()
+                tmp.pop("id")
+                tmp.pop("user")
+                music_detail_list.append(tmp)
+
+                if len(music_detail_list) == data["maxCount"]:
+                    next_index = data["maxCount"] + data["nextIndex"]
+                    break
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_index,
+            "userMusicList": [{"userMusicDetailList": music_detail_list}],
+        }
+
+    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+        if p is None:
+            return {}
+
+        return {
+            "userName": p["userName"],
+            "rating": p["playerRating"],
+            # hardcode lastDataVersion for CardMaker 1.34
+            "lastDataVersion": "1.20.00",
+            "isLogin": False,
+            "isExistSellingCard": False,
+        }
+
+    def handle_cm_get_user_data_api_request(self, data: Dict) -> Dict:
+        # user already exists, because the preview checks that already
+        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+
+        cards = self.data.card.get_user_cards(data["userId"])
+        if cards is None or len(cards) == 0:
+            # This should never happen
+            self.logger.error(
+                f"handle_get_user_data_api_request: Internal error - No cards found for user id {data['userId']}"
+            )
+            return {}
+
+        # get the dict representation of the row so we can modify values
+        user_data = p._asdict()
+
+        # remove the values the game doesn't want
+        user_data.pop("id")
+        user_data.pop("user")
+        user_data.pop("version")
+
+        return {"userId": data["userId"], "userData": user_data}
+
+    def handle_cm_login_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}
+
+    def handle_cm_logout_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}
+
+    def handle_cm_get_selling_card_api_request(self, data: Dict) -> Dict:
+        selling_cards = self.data.static.get_enabled_cards(self.version)
+        if selling_cards is None:
+            return {"length": 0, "sellingCardList": []}
+
+        selling_card_list = []
+        for card in selling_cards:
+            tmp = card._asdict()
+            tmp.pop("id")
+            tmp.pop("version")
+            tmp.pop("cardName")
+            tmp.pop("enabled")
+
+            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["noticeStartDate"] = datetime.strftime(
+                tmp["noticeStartDate"], "%Y-%m-%d %H:%M:%S"
+            )
+            tmp["noticeEndDate"] = datetime.strftime(
+                tmp["noticeEndDate"], "%Y-%m-%d %H:%M:%S"
+            )
+
+            selling_card_list.append(tmp)
+
+        return {"length": len(selling_card_list), "sellingCardList": selling_card_list}
+
+    def handle_cm_get_user_card_api_request(self, data: Dict) -> Dict:
+        user_cards = self.data.item.get_cards(data["userId"])
+        if user_cards is None:
+            return {"returnCode": 1, "length": 0, "nextIndex": 0, "userCardList": []}
+
+        max_ct = data["maxCount"]
+        next_idx = data["nextIndex"]
+        start_idx = next_idx
+        end_idx = max_ct + start_idx
+
+        if len(user_cards[start_idx:]) > max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        card_list = []
+        for card in user_cards:
+            tmp = card._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+
+            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+            card_list.append(tmp)
+
+        return {
+            "returnCode": 1,
+            "length": len(card_list[start_idx:end_idx]),
+            "nextIndex": next_idx,
+            "userCardList": card_list[start_idx:end_idx],
+        }
+
+    def handle_cm_get_user_item_api_request(self, data: Dict) -> Dict:
+        super().handle_get_user_item_api_request(data)
+
+    def handle_cm_get_user_character_api_request(self, data: Dict) -> Dict:
+        characters = self.data.item.get_characters(data["userId"])
+
+        chara_list = []
+        for chara in characters:
+            chara_list.append(
+                {
+                    "characterId": chara["characterId"],
+                    # no clue why those values are even needed
+                    "point": 0,
+                    "count": 0,
+                    "level": chara["level"],
+                    "nextAwake": 0,
+                    "nextAwakePercent": 0,
+                    "favorite": False,
+                    "awakening": chara["awakening"],
+                    "useCount": chara["useCount"],
+                }
+            )
+
+        return {
+            "returnCode": 1,
+            "length": len(chara_list),
+            "userCharacterList": chara_list,
+        }
+
+    def handle_cm_get_user_card_print_error_api_request(self, data: Dict) -> Dict:
+        return {"length": 0, "userPrintDetailList": []}
+
+    def handle_cm_upsert_user_print_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        upsert = data["userPrintDetail"]
+
+        # set a random card serial number
+        serial_id = "".join([str(randint(0, 9)) for _ in range(20)])
+
+        user_card = upsert["userCard"]
+        self.data.item.put_card(
+            user_id,
+            user_card["cardId"],
+            user_card["cardTypeId"],
+            user_card["charaId"],
+            user_card["mapId"],
+        )
+
+        # properly format userPrintDetail for the database
+        upsert.pop("userCard")
+        upsert.pop("serialId")
+        upsert["printDate"] = datetime.strptime(upsert["printDate"], "%Y-%m-%d")
+
+        self.data.item.put_user_print_detail(user_id, serial_id, upsert)
+
+        return {
+            "returnCode": 1,
+            "orderId": 0,
+            "serialId": serial_id,
+            "startDate": "2018-01-01 00:00:00",
+            "endDate": "2038-01-01 00:00:00",
+        }
+
+    def handle_cm_upsert_user_printlog_api_request(self, data: Dict) -> Dict:
+        return {
+            "returnCode": 1,
+            "orderId": 0,
+            "serialId": data["userPrintlog"]["serialId"],
+        }
+
+    def handle_cm_upsert_buy_card_api_request(self, data: Dict) -> Dict:
+        return {"returnCode": 1}

titles/mai2/dxplus.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2Plus(Mai2Base):
+class Mai2DXPlus(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_PLUS

titles/mai2/festival.py

@@ -1,12 +1,12 @@
 from typing import Dict
 
 from core.config import CoreConfig
-from titles.mai2.universeplus import Mai2UniversePlus
+from titles.mai2.dx import Mai2DX
 from titles.mai2.const import Mai2Constants
 from titles.mai2.config import Mai2Config
 
 
-class Mai2Festival(Mai2UniversePlus):
+class Mai2Festival(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_FESTIVAL

titles/mai2/finale.py

@@ -0,0 +1,23 @@
+from typing import Any, List, Dict
+from datetime import datetime, timedelta
+import pytz
+import json
+
+from core.config import CoreConfig
+from titles.mai2.base import Mai2Base
+from titles.mai2.config import Mai2Config
+from titles.mai2.const import Mai2Constants
+
+
+class Mai2Finale(Mai2Base):
+    def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
+        super().__init__(cfg, game_cfg)
+        self.version = Mai2Constants.VER_MAIMAI_FINALE
+        self.can_deliver = True
+        self.can_usbdl = True
+        
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/197/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/197/"

titles/mai2/index.py

@@ -1,4 +1,5 @@
 from twisted.web.http import Request
+from twisted.web.server import NOT_DONE_YET
 import json
 import inflection
 import yaml
@@ -14,7 +15,9 @@ from core.utils import Utils
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 from titles.mai2.base import Mai2Base
-from titles.mai2.plus import Mai2Plus
+from titles.mai2.finale import Mai2Finale
+from titles.mai2.dx import Mai2DX
+from titles.mai2.dxplus import Mai2DXPlus
 from titles.mai2.splash import Mai2Splash
 from titles.mai2.splashplus import Mai2SplashPlus
 from titles.mai2.universe import Mai2Universe
@@ -33,7 +36,20 @@ class Mai2Servlet:
 
         self.versions = [
             Mai2Base,
-            Mai2Plus,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            Mai2Finale,            
+            Mai2DX,
+            Mai2DXPlus,
             Mai2Splash,
             Mai2SplashPlus,
             Mai2Universe,
@@ -42,27 +58,29 @@ class Mai2Servlet:
         ]
 
         self.logger = logging.getLogger("mai2")
-        log_fmt_str = "[%(asctime)s] Mai2 | %(levelname)s | %(message)s"
+        if not hasattr(self.logger, "initted"):
-        log_fmt = logging.Formatter(log_fmt_str)
+            log_fmt_str = "[%(asctime)s] Mai2 | %(levelname)s | %(message)s"
-        fileHandler = TimedRotatingFileHandler(
+            log_fmt = logging.Formatter(log_fmt_str)
-            "{0}/{1}.log".format(self.core_cfg.server.log_dir, "mai2"),
+            fileHandler = TimedRotatingFileHandler(
-            encoding="utf8",
+                "{0}/{1}.log".format(self.core_cfg.server.log_dir, "mai2"),
-            when="d",
+                encoding="utf8",
-            backupCount=10,
+                when="d",
-        )
+                backupCount=10,
+            )
 
-        fileHandler.setFormatter(log_fmt)
+            fileHandler.setFormatter(log_fmt)
 
-        consoleHandler = logging.StreamHandler()
+            consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
+            consoleHandler.setFormatter(log_fmt)
 
-        self.logger.addHandler(fileHandler)
+            self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
+            self.logger.addHandler(consoleHandler)
 
-        self.logger.setLevel(self.game_cfg.server.loglevel)
+            self.logger.setLevel(self.game_cfg.server.loglevel)
-        coloredlogs.install(
+            coloredlogs.install(
-            level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
+                level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
+            )
+            self.logger.initted = True
 
     @classmethod
     def get_allnet_info(
@@ -82,7 +100,7 @@ class Mai2Servlet:
             return (
                 True,
                 f"http://{core_cfg.title.hostname}:{core_cfg.title.port}/{game_code}/$v/",
-                f"{core_cfg.title.hostname}:{core_cfg.title.port}",
+                f"{core_cfg.title.hostname}",
             )
 
         return (
@@ -95,6 +113,10 @@ class Mai2Servlet:
         if url_path.lower() == "ping":
             return zlib.compress(b'{"returnCode": "1"}')
         
+        elif url_path.startswith("api/movie/"):
+            self.logger.info(f"Movie data: {url_path} - {request.content.getvalue()}")
+            return b""
+
         req_raw = request.content.getvalue()
         url = request.uri.decode()
         url_split = url_path.split("/")
@@ -102,20 +124,49 @@ class Mai2Servlet:
         endpoint = url_split[len(url_split) - 1]
         client_ip = Utils.get_ip_addr(request)
 
-        if version < 105:  # 1.0
+        if request.uri.startswith(b"/SDEZ"):
-            internal_ver = Mai2Constants.VER_MAIMAI_DX
+            if version < 105:  # 1.0
-        elif version >= 105 and version < 110:  # Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_PLUS
+            elif version >= 105 and version < 110:  # Plus
-        elif version >= 110 and version < 115:  # Splash
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_PLUS
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH
+            elif version >= 110 and version < 115:  # Splash
-        elif version >= 115 and version < 120:  # Splash Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS
+            elif version >= 115 and version < 120:  # Splash Plus
-        elif version >= 120 and version < 125:  # Universe
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
+            elif version >= 120 and version < 125:  # Universe
-        elif version >= 125 and version < 130:  # Universe Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS
+            elif version >= 125 and version < 130:  # Universe Plus
-        elif version >= 130:  # Festival
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL
+            elif version >= 130:  # Festival
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL
+
+        else:
+            if version < 110:  # 1.0
+                internal_ver = Mai2Constants.VER_MAIMAI
+            elif version >= 110 and version < 120:  # Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_PLUS
+            elif version >= 120 and version < 130:  # Green
+                internal_ver = Mai2Constants.VER_MAIMAI_GREEN
+            elif version >= 130 and version < 140:  # Green Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_GREEN_PLUS
+            elif version >= 140 and version < 150:  # Orange
+                internal_ver = Mai2Constants.VER_MAIMAI_ORANGE
+            elif version >= 150 and version < 160:  # Orange Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_ORANGE_PLUS
+            elif version >= 160 and version < 170:  # Pink
+                internal_ver = Mai2Constants.VER_MAIMAI_PINK
+            elif version >= 170 and version < 180:  # Pink Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_PINK_PLUS
+            elif version >= 180 and version < 185:  # Murasaki
+                internal_ver = Mai2Constants.VER_MAIMAI_MURASAKI
+            elif version >= 185 and version < 190:  # Murasaki Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_MURASAKI_PLUS
+            elif version >= 190 and version < 195:  # Milk
+                internal_ver = Mai2Constants.VER_MAIMAI_MILK
+            elif version >= 195 and version < 197:  # Milk Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_MILK_PLUS
+            elif version >= 197:  # Finale
+                internal_ver = Mai2Constants.VER_MAIMAI_FINALE
         
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
             # If we get a 32 character long hex string, it's a hash and we're
@@ -159,3 +210,39 @@ class Mai2Servlet:
         self.logger.debug(f"Response {resp}")
 
         return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))
+
+    def render_GET(self, request: Request, version: int, url_path: str) -> bytes:
+        self.logger.info(f"v{version} GET {url_path}")
+        url_split = url_path.split("/")
+
+        if (url_split[0] == "api" and url_split[1] == "movie") or url_split[0] == "movie":
+            if url_split[2] == "moviestart":
+                return json.dumps({"moviestart":{"status":"OK"}}).encode()
+
+        if url_split[0] == "old":
+            if url_split[1] == "ping":
+                self.logger.info(f"v{version} old server ping")
+                return zlib.compress(b"ok")
+
+            elif url_split[1].startswith("userdata"):
+                self.logger.info(f"v{version} old server userdata inquire")
+                return zlib.compress(b"{}")
+
+            elif url_split[1].startswith("friend"):
+                self.logger.info(f"v{version} old server friend inquire")
+                return zlib.compress(b"{}")
+        
+        elif url_split[0] == "usbdl":
+            if url_split[1] == "CONNECTIONTEST":
+                self.logger.info(f"v{version} usbdl server test")
+                return zlib.compress(b"ok")
+
+        elif url_split[0] == "deliver":
+            file = url_split[len(url_split) - 1]
+            self.logger.info(f"v{version} {file} deliver inquire")
+            
+            if not self.game_cfg.deliver.enable or not path.exists(f"{self.game_cfg.deliver.content_folder}/{file}"):
+                return zlib.compress(b"")
+        
+        else:
+            return zlib.compress(b"{}")

titles/mai2/read.py

@@ -4,6 +4,9 @@ import os
 import re
 import xml.etree.ElementTree as ET
 from typing import Any, Dict, List, Optional
+from Crypto.Cipher import AES
+import zlib
+import codecs
 
 from core.config import CoreConfig
 from core.data import Data
@@ -34,18 +37,147 @@ class Mai2Reader(BaseReader):
 
     def read(self) -> None:
         data_dirs = []
-        if self.bin_dir is not None:
+        if self.version >= Mai2Constants.VER_MAIMAI_DX:
-            data_dirs += self.get_data_directories(self.bin_dir)
+            if self.bin_dir is not None:
+                data_dirs += self.get_data_directories(self.bin_dir)
 
-        if self.opt_dir is not None:
+            if self.opt_dir is not None:
-            data_dirs += self.get_data_directories(self.opt_dir)
+                data_dirs += self.get_data_directories(self.opt_dir)
 
-        for dir in data_dirs:
+            for dir in data_dirs:
-            self.logger.info(f"Read from {dir}")
+                self.logger.info(f"Read from {dir}")
-            self.get_events(f"{dir}/event")
+                self.get_events(f"{dir}/event")
-            self.disable_events(f"{dir}/information", f"{dir}/scoreRanking")
+                self.disable_events(f"{dir}/information", f"{dir}/scoreRanking")
-            self.read_music(f"{dir}/music")
+                self.read_music(f"{dir}/music")
-            self.read_tickets(f"{dir}/ticket")
+                self.read_tickets(f"{dir}/ticket")
+        
+        else:
+            if not os.path.exists(f"{self.bin_dir}/tables"):
+                self.logger.error(f"tables directory not found in {self.bin_dir}")
+                return
+            
+            if self.version >= Mai2Constants.VER_MAIMAI_MILK:
+                if self.extra is None:
+                    self.logger.error("Milk - Finale requre an AES key via a hex string send as the --extra flag")
+                    return
+                
+                key = bytes.fromhex(self.extra)
+            
+            else:
+                key = None
+            
+            evt_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmEvent.bin", key)
+            txt_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmtextout_jp.bin", key)
+            score_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmScore.bin", key)
+            
+            self.read_old_events(evt_table)
+            self.read_old_music(score_table, txt_table)
+            
+            if self.opt_dir is not None:
+                evt_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmEvent.bin", key)
+                txt_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmtextout_jp.bin", key)
+                score_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmScore.bin", key)
+
+                self.read_old_events(evt_table)
+                self.read_old_music(score_table, txt_table)
+
+            return
+    
+    def load_table_raw(self, dir: str, file: str, key: Optional[bytes]) -> Optional[List[Dict[str, str]]]:
+        if not os.path.exists(f"{dir}/{file}"):
+            self.logger.warn(f"file {file} does not exist in directory {dir}, skipping")
+            return
+        
+        self.logger.info(f"Load table {file} from {dir}")
+        if key is not None:
+            cipher = AES.new(key, AES.MODE_CBC)
+            with open(f"{dir}/{file}", "rb") as f:
+                f_encrypted = f.read()
+                f_data = cipher.decrypt(f_encrypted)[0x10:]
+        
+        else:
+            with open(f"{dir}/{file}", "rb") as f:
+                f_data = f.read()[0x10:]
+        
+        if f_data is None or not f_data:
+            self.logger.warn(f"file {dir} could not be read, skipping")
+            return
+        
+        f_data_deflate = zlib.decompress(f_data, wbits = zlib.MAX_WBITS | 16)[0x12:] # lop off the junk at the beginning
+        f_decoded = codecs.utf_16_le_decode(f_data_deflate)[0]
+        f_split = f_decoded.splitlines()
+
+        has_struct_def = "struct " in f_decoded
+        is_struct = False
+        struct_def = []
+        tbl_content = []
+
+        if has_struct_def:
+            for x in f_split:
+                if x.startswith("struct "):
+                    is_struct = True
+                    struct_name = x[7:-1]
+                    continue
+                
+                if x.startswith("};"):
+                    is_struct = False
+                    break
+
+                if is_struct:
+                    try:
+                        struct_def.append(x[x.rindex("  ") + 2: -1])
+                    except ValueError:
+                        self.logger.warn(f"rindex failed on line {x}")
+            
+            if is_struct:
+                self.logger.warn("Struct not formatted properly")
+            
+            if not struct_def:
+                self.logger.warn("Struct def not found")
+                    
+        name = file[:file.index(".")]
+        if "_" in name:
+            name = name[:file.index("_")]
+        
+        for x in f_split:
+            if not x.startswith(name.upper()):
+                continue
+            
+            line_match = re.match(r"(\w+)\((.*?)\)([ ]+\/{3}<[ ]+(.*))?", x)
+            if line_match is None:
+                continue
+
+            if not line_match.group(1) == name.upper():
+                self.logger.warn(f"Strange regex match for line {x} -> {line_match}")
+                continue
+            
+            vals = line_match.group(2)
+            comment = line_match.group(4)
+            line_dict = {}
+            
+            vals_split = vals.split(",")
+            for y in range(len(vals_split)):
+                stripped = vals_split[y].strip().lstrip("L\"").lstrip("\"").rstrip("\"")
+                if not stripped or stripped is None:
+                    continue
+
+                if has_struct_def and len(struct_def) > y:
+                    line_dict[struct_def[y]] = stripped
+                
+                else:
+                    line_dict[f'item_{y}'] = stripped
+            
+            if comment:
+                line_dict['comment'] = comment
+            
+            tbl_content.append(line_dict)
+
+        if tbl_content:
+            return tbl_content
+        
+        else:
+            self.logger.warning("Failed load table content, skipping")
+            return
 
     def get_events(self, base_dir: str) -> None:
         self.logger.info(f"Reading events from {base_dir}...")
@@ -188,3 +320,24 @@ class Mai2Reader(BaseReader):
                             self.version, id, ticket_type, price, name
                         )
                         self.logger.info(f"Added ticket {id}...")
+
+    def read_old_events(self, events: Optional[List[Dict[str, str]]]) -> None:
+        if events is None:
+            return
+        
+        for event in events:
+            evt_id = int(event.get('イベントID', '0'))
+            evt_expire_time = float(event.get('オフ時強制時期', '0.0'))
+            is_exp = bool(int(event.get('海外許可', '0')))
+            is_aou = bool(int(event.get('AOU許可', '0')))
+            name = event.get('comment', f'evt_{evt_id}')
+
+            self.data.static.put_game_event(self.version, 0, evt_id, name)
+            
+            if not (is_exp or is_aou):
+                self.data.static.toggle_game_event(self.version, evt_id, False)
+    
+    def read_old_music(self, scores: Optional[List[Dict[str, str]]], text: Optional[List[Dict[str, str]]]) -> None:
+        if scores is None or text is None:
+            return
+        # TODO

titles/mai2/schema/item.py

@@ -18,10 +18,11 @@ character = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("characterId", Integer, nullable=False),
+    Column("characterId", Integer),
-    Column("level", Integer, nullable=False, server_default="1"),
+    Column("level", Integer),
-    Column("awakening", Integer, nullable=False, server_default="0"),
+    Column("awakening", Integer),
-    Column("useCount", Integer, nullable=False, server_default="0"),
+    Column("useCount", Integer),
+    Column("point", Integer),
     UniqueConstraint("user", "characterId", name="mai2_item_character_uk"),
     mysql_charset="utf8mb4",
 )
@@ -35,12 +36,12 @@ card = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("cardId", Integer, nullable=False),
+    Column("cardId", Integer),
-    Column("cardTypeId", Integer, nullable=False),
+    Column("cardTypeId", Integer),
-    Column("charaId", Integer, nullable=False),
+    Column("charaId", Integer),
-    Column("mapId", Integer, nullable=False),
+    Column("mapId", Integer),
-    Column("startDate", TIMESTAMP, nullable=False, server_default=func.now()),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
-    Column("endDate", TIMESTAMP, nullable=False),
+    Column("endDate", TIMESTAMP),
     UniqueConstraint("user", "cardId", "cardTypeId", name="mai2_item_card_uk"),
     mysql_charset="utf8mb4",
 )
@@ -54,10 +55,10 @@ item = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("itemId", Integer, nullable=False),
+    Column("itemId", Integer),
-    Column("itemKind", Integer, nullable=False),
+    Column("itemKind", Integer),
-    Column("stock", Integer, nullable=False, server_default="1"),
+    Column("stock", Integer),
-    Column("isValid", Boolean, nullable=False, server_default="1"),
+    Column("isValid", Boolean),
     UniqueConstraint("user", "itemId", "itemKind", name="mai2_item_item_uk"),
     mysql_charset="utf8mb4",
 )
@@ -71,11 +72,11 @@ map = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("mapId", Integer, nullable=False),
+    Column("mapId", Integer),
-    Column("distance", Integer, nullable=False),
+    Column("distance", Integer),
-    Column("isLock", Boolean, nullable=False, server_default="0"),
+    Column("isLock", Boolean),
-    Column("isClear", Boolean, nullable=False, server_default="0"),
+    Column("isClear", Boolean),
-    Column("isComplete", Boolean, nullable=False, server_default="0"),
+    Column("isComplete", Boolean),
     UniqueConstraint("user", "mapId", name="mai2_item_map_uk"),
     mysql_charset="utf8mb4",
 )
@@ -89,10 +90,10 @@ login_bonus = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("bonusId", Integer, nullable=False),
+    Column("bonusId", Integer),
-    Column("point", Integer, nullable=False),
+    Column("point", Integer),
-    Column("isCurrent", Boolean, nullable=False, server_default="0"),
+    Column("isCurrent", Boolean),
-    Column("isComplete", Boolean, nullable=False, server_default="0"),
+    Column("isComplete", Boolean),
     UniqueConstraint("user", "bonusId", name="mai2_item_login_bonus_uk"),
     mysql_charset="utf8mb4",
 )
@@ -106,12 +107,12 @@ friend_season_ranking = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("seasonId", Integer, nullable=False),
+    Column("seasonId", Integer),
-    Column("point", Integer, nullable=False),
+    Column("point", Integer),
-    Column("rank", Integer, nullable=False),
+    Column("rank", Integer),
-    Column("rewardGet", Boolean, nullable=False),
+    Column("rewardGet", Boolean),
-    Column("userName", String(8), nullable=False),
+    Column("userName", String(8)),
-    Column("recordDate", TIMESTAMP, nullable=False),
+    Column("recordDate", TIMESTAMP),
     UniqueConstraint(
         "user", "seasonId", "userName", name="mai2_item_friend_season_ranking_uk"
     ),
@@ -127,7 +128,7 @@ favorite = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("itemKind", Integer, nullable=False),
+    Column("itemKind", Integer),
     Column("itemIdList", JSON),
     UniqueConstraint("user", "itemKind", name="mai2_item_favorite_uk"),
     mysql_charset="utf8mb4",
@@ -142,10 +143,10 @@ charge = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("chargeId", Integer, nullable=False),
+    Column("chargeId", Integer),
-    Column("stock", Integer, nullable=False),
+    Column("stock", Integer),
-    Column("purchaseDate", String(255), nullable=False),
+    Column("purchaseDate", String(255)),
-    Column("validDate", String(255), nullable=False),
+    Column("validDate", String(255)),
     UniqueConstraint("user", "chargeId", name="mai2_item_charge_uk"),
     mysql_charset="utf8mb4",
 )
@@ -161,11 +162,11 @@ print_detail = Table(
     ),
     Column("orderId", Integer),
     Column("printNumber", Integer),
-    Column("printDate", TIMESTAMP, nullable=False, server_default=func.now()),
+    Column("printDate", TIMESTAMP, server_default=func.now()),
-    Column("serialId", String(20), nullable=False),
+    Column("serialId", String(20)),
-    Column("placeId", Integer, nullable=False),
+    Column("placeId", Integer),
-    Column("clientId", String(11), nullable=False),
+    Column("clientId", String(11)),
-    Column("printerSerialId", String(20), nullable=False),
+    Column("printerSerialId", String(20)),
     Column("cardRomVersion", Integer),
     Column("isHolograph", Boolean, server_default="1"),
     Column("printOption1", Boolean, server_default="0"),
@@ -333,6 +334,19 @@ class Mai2ItemData(BaseData):
             return None
         return result.fetchone()
     
+    def put_character_(self, user_id: int, char_data: Dict) -> Optional[int]:
+        char_data["user"] = user_id
+        sql = insert(character).values(**char_data)
+
+        conflict = sql.on_duplicate_key_update(**char_data)
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_character_: failed to insert item! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+
     def put_character(
         self,
         user_id: int,

titles/mai2/schema/profile.py

@@ -99,6 +99,68 @@ detail = Table(
     mysql_charset="utf8mb4",
 )
 
+detail_old = Table(
+   "maimai_profile_detail",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("lastDataVersion", Integer),
+    Column("userName", String(8)),
+    Column("point", Integer),
+    Column("totalPoint", Integer),
+    Column("iconId", Integer),
+    Column("nameplateId", Integer),
+    Column("frameId", Integer),
+    Column("trophyId", Integer),
+    Column("playCount", Integer),
+    Column("playVsCount", Integer),
+    Column("playSyncCount", Integer),
+    Column("winCount", Integer),
+    Column("helpCount", Integer),
+    Column("comboCount", Integer),
+    Column("feverCount", Integer),
+    Column("totalHiScore", Integer),
+    Column("totalEasyHighScore", Integer),
+    Column("totalBasicHighScore", Integer),
+    Column("totalAdvancedHighScore", Integer),
+    Column("totalExpertHighScore", Integer),
+    Column("totalMasterHighScore", Integer),
+    Column("totalReMasterHighScore", Integer),
+    Column("totalHighSync", Integer),
+    Column("totalEasySync", Integer),
+    Column("totalBasicSync", Integer),
+    Column("totalAdvancedSync", Integer),
+    Column("totalExpertSync", Integer),
+    Column("totalMasterSync", Integer),
+    Column("totalReMasterSync", Integer),
+    Column("playerRating", Integer),
+    Column("highestRating", Integer),
+    Column("rankAuthTailId", Integer),
+    Column("eventWatchedDate", String(255)),
+    Column("webLimitDate", String(255)),
+    Column("challengeTrackPhase", Integer),
+    Column("firstPlayBits", Integer),
+    Column("lastPlayDate", String(255)),
+    Column("lastPlaceId", Integer),
+    Column("lastPlaceName", String(255)),
+    Column("lastRegionId", Integer),
+    Column("lastRegionName", String(255)),
+    Column("lastClientId", String(255)),
+    Column("lastCountryCode", String(255)),
+    Column("eventPoint", Integer),
+    Column("totalLv", Integer),
+    Column("lastLoginBonusDay", Integer),
+    Column("lastSurvivalBonusDay", Integer),
+    Column("loginBonusLv", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_detail_uk"),
+    mysql_charset="utf8mb4",
+)
+
 ghost = Table(
     "mai2_profile_ghost",
     metadata,
@@ -223,6 +285,99 @@ option = Table(
     mysql_charset="utf8mb4",
 )
 
+option_old = Table(
+    "maimai_profile_option",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("soudEffect", Integer),
+    Column("mirrorMode", Integer),
+    Column("guideSpeed", Integer),
+    Column("bgInfo", Integer),
+    Column("brightness", Integer),
+    Column("isStarRot", Integer),
+    Column("breakSe", Integer),
+    Column("slideSe", Integer),
+    Column("hardJudge", Integer),
+    Column("isTagJump", Integer),
+    Column("breakSeVol", Integer),
+    Column("slideSeVol", Integer),
+    Column("isUpperDisp", Integer),
+    Column("trackSkip", Integer),
+    Column("optionMode", Integer),
+    Column("simpleOptionParam", Integer),
+    Column("adjustTiming", Integer),
+    Column("dispTiming", Integer),
+    Column("timingPos", Integer),
+    Column("ansVol", Integer),
+    Column("noteVol", Integer),
+    Column("dmgVol", Integer),
+    Column("appealFlame", Integer),
+    Column("isFeverDisp", Integer),
+    Column("dispJudge", Integer),
+    Column("judgePos", Integer),
+    Column("ratingGuard", Integer),
+    Column("selectChara", Integer),
+    Column("sortType", Integer),
+    Column("filterGenre", Integer),
+    Column("filterLevel", Integer),
+    Column("filterRank", Integer),
+    Column("filterVersion", Integer),
+    Column("filterRec", Integer),
+    Column("filterFullCombo", Integer),
+    Column("filterAllPerfect", Integer),
+    Column("filterDifficulty", Integer),
+    Column("filterFullSync", Integer),
+    Column("filterReMaster", Integer),
+    Column("filterMaxFever", Integer),
+    Column("finalSelectId", Integer),
+    Column("finalSelectCategory", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_option_uk"),
+    mysql_charset="utf8mb4",
+)
+
+web_opt = Table(
+    "maimai_profile_web_option",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("isNetMember", Boolean),
+    Column("dispRate", Integer),
+    Column("dispJudgeStyle", Integer),
+    Column("dispRank", Integer),
+    Column("dispHomeRanker", Integer),
+    Column("dispTotalLv", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_web_option_uk"),
+    mysql_charset="utf8mb4",
+)
+
+grade_status = Table(
+    "maimai_profile_grade_status",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("gradeVersion", Integer),
+    Column("gradeLevel", Integer),
+    Column("gradeSubLevel", Integer),
+    Column("gradeMaxId", Integer),
+    UniqueConstraint("user", "gradeVersion", name="maimai_profile_grade_status_uk"),
+    mysql_charset="utf8mb4",
+)
+
 rating = Table(
     "mai2_profile_rating",
     metadata,
@@ -268,42 +423,91 @@ activity = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("kind", Integer, nullable=False),
+    Column("kind", Integer),
-    Column("activityId", Integer, nullable=False),
+    Column("activityId", Integer),
-    Column("param1", Integer, nullable=False),
+    Column("param1", Integer),
-    Column("param2", Integer, nullable=False),
+    Column("param2", Integer),
-    Column("param3", Integer, nullable=False),
+    Column("param3", Integer),
-    Column("param4", Integer, nullable=False),
+    Column("param4", Integer),
-    Column("sortNumber", Integer, nullable=False),
+    Column("sortNumber", Integer),
     UniqueConstraint("user", "kind", "activityId", name="mai2_profile_activity_uk"),
     mysql_charset="utf8mb4",
 )
 
+boss = Table(
+    "maimai_profile_boss",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("pandoraFlagList0", Integer),
+    Column("pandoraFlagList1", Integer),
+    Column("pandoraFlagList2", Integer),
+    Column("pandoraFlagList3", Integer),
+    Column("pandoraFlagList4", Integer),
+    Column("pandoraFlagList5", Integer),
+    Column("pandoraFlagList6", Integer),
+    Column("emblemFlagList", Integer),
+    UniqueConstraint("user", name="mai2_profile_boss_uk"),
+    mysql_charset="utf8mb4",
+)
+
+recent_rating = Table(
+    "maimai_profile_recent_rating",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("userRecentRatingList", JSON),
+    UniqueConstraint("user", name="mai2_profile_recent_rating_uk"),
+    mysql_charset="utf8mb4",
+)
+
+consec_logins = Table(
+        "mai2_profile_consec_logins",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("version", Integer, nullable=False),
+    Column("logins", Integer),
+    UniqueConstraint("user", "version", name="mai2_profile_consec_logins_uk"),
+    mysql_charset="utf8mb4",
+)
 
 class Mai2ProfileData(BaseData):
     def put_profile_detail(
-        self, user_id: int, version: int, detail_data: Dict
+        self, user_id: int, version: int, detail_data: Dict, is_dx: bool = True
     ) -> Optional[Row]:
         detail_data["user"] = user_id
         detail_data["version"] = version
-        sql = insert(detail).values(**detail_data)
+
+        if is_dx:
+            sql = insert(detail).values(**detail_data)
+        else:
+            sql = insert(detail_old).values(**detail_data)
 
         conflict = sql.on_duplicate_key_update(**detail_data)
 
         result = self.execute(conflict)
         if result is None:
             self.logger.warn(
-                f"put_profile: Failed to create profile! user_id {user_id}"
+                f"put_profile: Failed to create profile! user_id {user_id} is_dx {is_dx}"
             )
             return None
         return result.lastrowid
 
-    def get_profile_detail(self, user_id: int, version: int) -> Optional[Row]:
+    def get_profile_detail(self, user_id: int, version: int, is_dx: bool = True) -> Optional[Row]:
-        sql = (
+        if is_dx:
-            select(detail)
+            sql = (
-            .where(and_(detail.c.user == user_id, detail.c.version <= version))
+                select(detail)
-            .order_by(detail.c.version.desc())
+                .where(and_(detail.c.user == user_id, detail.c.version <= version))
-        )
+                .order_by(detail.c.version.desc())
+            )
+        
+        else:
+            sql = (
+                select(detail_old)
+                .where(and_(detail_old.c.user == user_id, detail_old.c.version <= version))
+                .order_by(detail_old.c.version.desc())
+            )
 
         result = self.execute(sql)
         if result is None:
@@ -365,26 +569,36 @@ class Mai2ProfileData(BaseData):
         return result.fetchone()
 
     def put_profile_option(
-        self, user_id: int, version: int, option_data: Dict
+        self, user_id: int, version: int, option_data: Dict, is_dx: bool = True
     ) -> Optional[int]:
         option_data["user"] = user_id
         option_data["version"] = version
 
-        sql = insert(option).values(**option_data)
+        if is_dx:
+            sql = insert(option).values(**option_data)
+        else:
+            sql = insert(option_old).values(**option_data)
         conflict = sql.on_duplicate_key_update(**option_data)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"put_profile_option: failed to update! {user_id}")
+            self.logger.warn(f"put_profile_option: failed to update! {user_id} is_dx {is_dx}")
             return None
         return result.lastrowid
 
-    def get_profile_option(self, user_id: int, version: int) -> Optional[Row]:
+    def get_profile_option(self, user_id: int, version: int, is_dx: bool = True) -> Optional[Row]:
-        sql = (
+        if is_dx:
-            select(option)
+            sql = (
-            .where(and_(option.c.user == user_id, option.c.version <= version))
+                select(option)
-            .order_by(option.c.version.desc())
+                .where(and_(option.c.user == user_id, option.c.version <= version))
-        )
+                .order_by(option.c.version.desc())
+            )
+        else:
+            sql = (
+                select(option_old)
+                .where(and_(option_old.c.user == user_id, option_old.c.version <= version))
+                .order_by(option_old.c.version.desc())
+            )
 
         result = self.execute(sql)
         if result is None:
@@ -474,3 +688,130 @@ class Mai2ProfileData(BaseData):
         if result is None:
             return None
         return result.fetchall()
+
+    def put_web_option(self, user_id: int, version: int, web_opts: Dict) -> Optional[int]:
+        web_opts["user"] = user_id
+        web_opts["version"] = version
+        sql = insert(web_opt).values(**web_opts)
+
+        conflict = sql.on_duplicate_key_update(**web_opts)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_web_option: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_web_option(self, user_id: int, version: int) -> Optional[Row]:
+        sql = web_opt.select(and_(web_opt.c.user == user_id, web_opt.c.version == version))
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_grade_status(self, user_id: int, grade_stat: Dict) -> Optional[int]:
+        grade_stat["user"] = user_id
+        sql = insert(grade_status).values(**grade_stat)
+
+        conflict = sql.on_duplicate_key_update(**grade_stat)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_grade_status: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_grade_status(self, user_id: int) -> Optional[Row]:
+        sql = grade_status.select(grade_status.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_boss_list(self, user_id: int, boss_stat: Dict) -> Optional[int]:
+        boss_stat["user"] = user_id
+        sql = insert(boss).values(**boss_stat)
+
+        conflict = sql.on_duplicate_key_update(**boss_stat)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_boss_list: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_boss_list(self, user_id: int) -> Optional[Row]:
+        sql = boss.select(boss.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_recent_rating(self, user_id: int, rr: Dict) -> Optional[int]:
+        sql = insert(recent_rating).values(user=user_id, userRecentRatingList=rr)
+
+        conflict = sql.on_duplicate_key_update({'userRecentRatingList': rr})
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warn(
+                f"put_recent_rating: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_recent_rating(self, user_id: int) -> Optional[Row]:
+        sql = recent_rating.select(recent_rating.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def add_consec_login(self, user_id: int, version: int) -> None:
+        sql = insert(consec_logins).values(
+            user=user_id,
+            version=version,
+            logins=1
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            logins=consec_logins.c.logins + 1
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to update consecutive login count for user {user_id} version {version}")
+
+    def get_consec_login(self, user_id: int, version: int) -> Optional[Row]:
+        sql = select(consec_logins).where(and_(
+            consec_logins.c.user==user_id,
+            consec_logins.c.version==version,
+        ))
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def reset_consec_login(self, user_id: int, version: int) -> Optional[Row]:
+        sql = consec_logins.update(and_(
+            consec_logins.c.user==user_id,
+            consec_logins.c.version==version,
+        )).values(
+            logins=1
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()

titles/mai2/schema/score.py

@@ -175,30 +175,137 @@ course = Table(
     mysql_charset="utf8mb4",
 )
 
+playlog_old = Table(
+    "maimai_playlog",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer),
+    # Pop access code
+    Column("orderId", Integer),
+    Column("sortNumber", Integer),
+    Column("placeId", Integer),
+    Column("placeName", String(255)),
+    Column("country", String(255)),
+    Column("regionId", Integer),
+    Column("playDate", String(255)),
+    Column("userPlayDate", String(255)),
+    Column("musicId", Integer),
+    Column("level", Integer),
+    Column("gameMode", Integer),
+    Column("rivalNum", Integer),
+    Column("track", Integer),
+    Column("eventId", Integer),
+    Column("isFreeToPlay", Boolean),
+    Column("playerRating", Integer),
+    Column("playedUserId1", Integer),
+    Column("playedUserId2", Integer),
+    Column("playedUserId3", Integer),
+    Column("playedUserName1", String(255)),
+    Column("playedUserName2", String(255)),
+    Column("playedUserName3", String(255)),
+    Column("playedMusicLevel1", Integer),
+    Column("playedMusicLevel2", Integer),
+    Column("playedMusicLevel3", Integer),
+    Column("achievement", Integer),
+    Column("score", Integer),
+    Column("tapScore", Integer),
+    Column("holdScore", Integer),
+    Column("slideScore", Integer),
+    Column("breakScore", Integer),
+    Column("syncRate", Integer),
+    Column("vsWin", Integer),
+    Column("isAllPerfect", Boolean),
+    Column("fullCombo", Integer),
+    Column("maxFever", Integer),
+    Column("maxCombo", Integer),
+    Column("tapPerfect", Integer),
+    Column("tapGreat", Integer),
+    Column("tapGood", Integer),
+    Column("tapBad", Integer),
+    Column("holdPerfect", Integer),
+    Column("holdGreat", Integer),
+    Column("holdGood", Integer),
+    Column("holdBad", Integer),
+    Column("slidePerfect", Integer),
+    Column("slideGreat", Integer),
+    Column("slideGood", Integer),
+    Column("slideBad", Integer),
+    Column("breakPerfect", Integer),
+    Column("breakGreat", Integer),
+    Column("breakGood", Integer),
+    Column("breakBad", Integer),
+    Column("judgeStyle", Integer),
+    Column("isTrackSkip", Boolean),
+    Column("isHighScore", Boolean),
+    Column("isChallengeTrack", Boolean),
+    Column("challengeLife", Integer),
+    Column("challengeRemain", Integer),
+    Column("isAllPerfectPlus", Integer),
+    mysql_charset="utf8mb4",
+)
+
+best_score_old = Table(
+    "maimai_score_best",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("musicId", Integer),
+    Column("level", Integer),
+    Column("playCount", Integer),
+    Column("achievement", Integer),
+    Column("scoreMax", Integer),
+    Column("syncRateMax", Integer),
+    Column("isAllPerfect", Boolean),
+    Column("isAllPerfectPlus", Integer),
+    Column("fullCombo", Integer),
+    Column("maxFever", Integer),
+    UniqueConstraint("user", "musicId", "level", name="maimai_score_best_uk"),
+    mysql_charset="utf8mb4",
+)
 
 class Mai2ScoreData(BaseData):
-    def put_best_score(self, user_id: int, score_data: Dict) -> Optional[int]:
+    def put_best_score(self, user_id: int, score_data: Dict, is_dx: bool = True) -> Optional[int]:
         score_data["user"] = user_id
-        sql = insert(best_score).values(**score_data)
 
+        if is_dx:
+            sql = insert(best_score).values(**score_data)
+        else:
+            sql = insert(best_score_old).values(**score_data)
         conflict = sql.on_duplicate_key_update(**score_data)
 
         result = self.execute(conflict)
         if result is None:
             self.logger.error(
-                f"put_best_score:  Failed to insert best score! user_id {user_id}"
+                f"put_best_score:  Failed to insert best score! user_id {user_id} is_dx {is_dx}"
             )
             return None
         return result.lastrowid
 
     @cached(2)
-    def get_best_scores(self, user_id: int, song_id: int = None) -> Optional[List[Row]]:
+    def get_best_scores(self, user_id: int, song_id: int = None, is_dx: bool = True) -> Optional[List[Row]]:
-        sql = best_score.select(
+        if is_dx:
-            and_(
+            sql = best_score.select(
-                best_score.c.user == user_id,
+                and_(
-                (best_score.c.song_id == song_id) if song_id is not None else True,
+                    best_score.c.user == user_id,
+                    (best_score.c.song_id == song_id) if song_id is not None else True,
+                )
+            )
+        else:
+            sql = best_score_old.select(
+                and_(
+                    best_score_old.c.user == user_id,
+                    (best_score_old.c.song_id == song_id) if song_id is not None else True,
+                )
             )
-        )
 
         result = self.execute(sql)
         if result is None:
@@ -221,15 +328,19 @@ class Mai2ScoreData(BaseData):
             return None
         return result.fetchone()
 
-    def put_playlog(self, user_id: int, playlog_data: Dict) -> Optional[int]:
+    def put_playlog(self, user_id: int, playlog_data: Dict, is_dx: bool = True) -> Optional[int]:
         playlog_data["user"] = user_id
-        sql = insert(playlog).values(**playlog_data)
+
+        if is_dx:
+            sql = insert(playlog).values(**playlog_data)
+        else:
+            sql = insert(playlog_old).values(**playlog_data)
 
         conflict = sql.on_duplicate_key_update(**playlog_data)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.error(f"put_playlog:  Failed to insert! user_id {user_id}")
+            self.logger.error(f"put_playlog:  Failed to insert! user_id {user_id} is_dx {is_dx}")
             return None
         return result.lastrowid
 
@@ -251,4 +362,4 @@ class Mai2ScoreData(BaseData):
         result = self.execute(sql)
         if result is None:
             return None
-        return result.fetchone()
+        return result.fetchall()

titles/mai2/splash.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2Splash(Mai2Base):
+class Mai2Splash(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_SPLASH

titles/mai2/splashplus.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2SplashPlus(Mai2Base):
+class Mai2SplashPlus(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS

titles/mai2/universe.py

@@ -5,12 +5,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.const import Mai2Constants
 from titles.mai2.config import Mai2Config
 
 
-class Mai2Universe(Mai2Base):
+class Mai2Universe(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_UNIVERSE

titles/mai2/universeplus.py

@@ -1,12 +1,12 @@
 from typing import Dict
 
 from core.config import CoreConfig
-from titles.mai2.universe import Mai2Universe
+from titles.mai2.dx import Mai2DX
 from titles.mai2.const import Mai2Constants
 from titles.mai2.config import Mai2Config
 
 
-class Mai2UniversePlus(Mai2Universe):
+class Mai2UniversePlus(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS

titles/pokken/base.py

@@ -8,6 +8,7 @@ from core import CoreConfig
 from .config import PokkenConfig
 from .proto import jackal_pb2
 from .database import PokkenData
+from .const import PokkenConstants
 
 
 class PokkenBase:
@@ -301,11 +302,11 @@ class PokkenBase:
         battle = req.battle_data
         mon = req.pokemon_data
 
-        self.data.profile.update_support_team(user_id, 1, req.support_set_1[0], req.support_set_1[1])
+        p = self.data.profile.touch_profile(user_id)
-        self.data.profile.update_support_team(user_id, 2, req.support_set_2[0], req.support_set_2[1])
+        if p is None or not p:
-        self.data.profile.update_support_team(user_id, 3, req.support_set_3[0], req.support_set_3[1])
+            self.data.profile.create_profile(user_id)
 
-        if req.trainer_name_pending: # we're saving for the first time
+        if req.trainer_name_pending is not None and req.trainer_name_pending: # we're saving for the first time
             self.data.profile.set_profile_name(user_id, req.trainer_name_pending, req.avatar_gender if req.avatar_gender else None)
 
         for tut_flg in req.tutorial_progress_flag:
@@ -329,6 +330,46 @@ class PokkenBase:
         for reward in req.reward_data:
             self.data.item.add_reward(user_id, reward.get_category_id, reward.get_content_id, reward.get_type_id)
         
+        self.data.profile.add_profile_points(user_id, get_rank_pts, get_money, get_score_pts, grade_max)
+        
+        self.data.profile.update_support_team(user_id, 1, req.support_set_1[0], req.support_set_1[1])
+        self.data.profile.update_support_team(user_id, 2, req.support_set_2[0], req.support_set_2[1])
+        self.data.profile.update_support_team(user_id, 3, req.support_set_3[0], req.support_set_3[1])
+
+        self.data.profile.put_pokemon(user_id, mon.char_id, mon.illustration_book_no, mon.bp_point_atk, mon.bp_point_res, mon.bp_point_def, mon.bp_point_sp)
+        self.data.profile.add_pokemon_xp(user_id, mon.char_id, mon.get_pokemon_exp)
+        
+        for x in range(len(battle.play_mode)):
+            self.data.profile.put_pokemon_battle_result(
+                user_id, 
+                mon.char_id, 
+                PokkenConstants.BATTLE_TYPE(battle.play_mode[x]), 
+                PokkenConstants.BATTLE_RESULT(battle.result[x])
+            )
+
+        self.data.profile.put_stats(
+            user_id,
+            battle.ex_ko_num,
+            battle.wko_num,
+            battle.timeup_win_num,
+            battle.cool_ko_num,
+            battle.perfect_ko_num,
+            num_continues
+        )
+
+        self.data.profile.put_extra(
+            user_id,
+            extra_counter,
+            evt_reward_get_flg,
+            total_play_days,
+            awake_num,
+            use_support_ct,
+            beat_num,
+            aid_skill,
+            last_evt
+        )
+
+
         return res.SerializeToString()
 
     def handle_save_ingame_log(self, data: jackal_pb2.Request) -> bytes:

titles/pokken/schema/profile.py

@@ -137,6 +137,14 @@ pokemon_data = Table(
 
 
 class PokkenProfileData(BaseData):
+    def touch_profile(self, user_id: int) -> Optional[int]:
+        sql = select([profile.c.id]).where(profile.c.user == user_id)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()['id']
+
     def create_profile(self, user_id: int) -> Optional[int]:
         sql = insert(profile).values(user=user_id)
         conflict = sql.on_duplicate_key_update(user=user_id)
@@ -158,6 +166,33 @@ class PokkenProfileData(BaseData):
                 f"Failed to update pokken profile name for user {user_id}!"
             )
 
+    def put_extra(
+        self, 
+        user_id: int, 
+        extra_counter: int,
+        evt_reward_get_flg: int,
+        total_play_days: int,
+        awake_num: int,
+        use_support_ct: int,
+        beat_num: int,
+        aid_skill: int,
+        last_evt: int
+    ) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            extra_counter=extra_counter,
+            event_reward_get_flag=evt_reward_get_flg,
+            total_play_days=total_play_days,
+            awake_num=awake_num,
+            use_support_num=use_support_ct,
+            beat_num=beat_num,
+            aid_skill=aid_skill,
+            last_play_event_id=last_evt
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to put extra data for user {user_id}")
+
     def update_profile_tutorial_flags(self, user_id: int, tutorial_flags: List) -> None:
         sql = update(profile).where(profile.c.user == user_id).values(
             tutorial_progress_flag=tutorial_flags,
@@ -192,9 +227,14 @@ class PokkenProfileData(BaseData):
             )
 
     def add_profile_points(
-        self, user_id: int, rank_pts: int, money: int, score_pts: int
+        self, user_id: int, rank_pts: int, money: int, score_pts: int, grade_max: int
     ) -> None:
-        pass
+        sql = update(profile).where(profile.c.user == user_id).values(
+            trainer_rank_point = profile.c.trainer_rank_point + rank_pts,
+            fight_money = profile.c.fight_money + money,
+            score_point = profile.c.score_point + score_pts,
+            grade_max_num = grade_max
+        )
 
     def get_profile(self, user_id: int) -> Optional[Row]:
         sql = profile.select(profile.c.user == user_id)
@@ -294,7 +334,18 @@ class PokkenProfileData(BaseData):
         """
         Records profile stats
         """
-        pass
+        sql = update(profile).where(profile.c.user==user_id).values(
+            ex_ko_num=profile.c.ex_ko_num + exkos,
+            wko_num=profile.c.wko_num + wkos,
+            timeup_win_num=profile.c.timeup_win_num + timeout_wins,
+            cool_ko_num=profile.c.cool_ko_num + cool_kos,
+            perfect_ko_num=profile.c.perfect_ko_num + perfects,
+            continue_num=continues,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to update stats for user {user_id}")
 
     def update_support_team(self, user_id: int, support_id: int, support1: int = 4294967295, support2: int = 4294967295) -> None:
         sql = update(profile).where(profile.c.user==user_id).values(