README.md

@@ -1,6 +1,6 @@
 # AQUA-to-ARTEMiS
 
-Imports AQUAs db.sqlite or MySQL database into ARTEMiS, currently WIP
+Imports AQUAs db.sqlite database into ARTEMiS database, currently WIP
 
 ## Requirements
 
@@ -17,19 +17,19 @@ python dbutils.py create
 Place `aqua_importer.py` in your ARTEMiS directory and execute it:
 
 ```sh
-python aqua_importer.py "C:\path\to\aqua"
+python aqua_importer.py "C:\path\to\aqua\data"
 ```
 
 **CAN OVERWRITE ALREADY EXISTING GAMES DATA! BE WARNED!!**
 
 ## How to use
 ```
-usage: aqua_importer.py [-h] [--config CONFIG] aqua_data_path
+usage: aqua_importer.py [-h] [--config CONFIG] aqua_folder_path
 
 AQUA to ARTEMiS
 
 positional arguments:
-  aqua_data_path      Absolute folder path to AQUA folder, where application.properties is located in.
+  aqua_folder_path      Absolute folder path to AQUA /data folder, where db.sqlite is located in
 
 options:
   -h, --help            show this help message and exit
@@ -41,6 +41,6 @@ options:
 
 - [x] CHUNITHM SUN
 - [x] O.N.G.E.K.I. Bright Memory
-- [x] maimai DX FESTiVAL
+- [ ] maimai DX UNiVERSE/FESTiVAL
 - [ ] Change the game version to import using arguments
 - [ ] Code clean up/optimizations

aqua_importer.py

@@ -10,6 +10,7 @@ from sqlalchemy.exc import SQLAlchemyError
 from logging.handlers import TimedRotatingFileHandler
 from typing import Any, Dict, Optional
 import yaml
+import yaml
 import argparse
 import logging
 import coloredlogs
@@ -22,19 +23,13 @@ from titles.chuni.sun import ChuniSun
 from titles.ongeki.brightmemory import OngekiBrightMemory
 from titles.ongeki.config import OngekiConfig
 from titles.ongeki.const import OngekiConstants
-from titles.mai2.festival import Mai2Festival
-from titles.mai2.config import Mai2Config
-from titles.mai2.const import Mai2Constants
 
 
 class AquaData:
     def __init__(self, aqua_db_path: str) -> None:
-        if "@" in aqua_db_path:
-            self.__url = f"mysql+pymysql://{aqua_db_path}"
-        else:
-            self.__url = f"sqlite:///{aqua_db_path}"
+        self.__url = f"sqlite:///{aqua_db_path}"
 
-        self.__engine = create_engine(self.__url, pool_recycle=3600, echo=False)
+        self.__engine = create_engine(self.__url, pool_recycle=3600)
         # self.inspector = reflection.Inspector.from_engine(self.__engine)
 
         session = sessionmaker(bind=self.__engine)
@@ -96,7 +91,6 @@ class Importer:
         self.config_folder = cfg_folder
         self.data = Data(core_cfg)
         self.title_registry: Dict[str, Any] = {}
-        self.use_mysql = False
 
         self.logger = logging.getLogger("importer")
         if not hasattr(self.logger, "initialized"):
@@ -120,49 +114,17 @@ class Importer:
             coloredlogs.install(level="INFO", logger=self.logger, fmt=log_fmt_str)
             self.logger.initialized = True
 
-        if not os.path.isfile(f"{aqua_folder}/application.properties"):
-            self.logger.error("Could not locate AQUA application.properties file!")
-            exit(1)
-
-        with open(f"{aqua_folder}/application.properties") as file:
-            lines = file.readlines()
-
-        properties = {}
-        for line in lines:
-            line = line.strip()
-            if not line or line.startswith("#"):
-                continue
-            parts = line.split("=")
-            if len(parts) >= 2:
-                key = parts[0].strip()
-                value = "=".join(parts[1:]).strip()
-                properties[key] = value
-
-        db_driver = properties.get("spring.datasource.driver-class-name")
-        if "sqlite" in db_driver:
-            aqua_db_path = None
-            db_url = properties.get("spring.datasource.url").split("sqlite:")[1]
-            temp = os.path.join(f"{aqua_folder}/{db_url}")
+        aqua_db_path = None
+        if os.path.exists(aqua_folder):
+            temp = os.path.join(aqua_folder, "db.sqlite")
             if os.path.isfile(temp):
                 aqua_db_path = temp
 
-            if not aqua_db_path:
-                self.logger.error("Could not locate AQUA db.sqlite file!")
-                exit(1)
+        if not aqua_db_path:
+            self.logger.error("Could not locate AQUA db.sqlite file!")
+            exit(1)
 
-            self.aqua = AquaData(aqua_db_path)
-
-        elif "mysql" in db_driver or "mariadb" in db_driver:
-            self.use_mysql = True
-            db_username = properties.get("spring.datasource.username")
-            db_password = properties.get("spring.datasource.password")
-            db_url = (
-                properties.get("spring.datasource.url").split("?")[0].split("//")[1]
-            )
-            self.aqua = AquaData(f"{db_username}:{db_password}@{db_url}")
-
-        else:
-            self.logger.error("Unknown database type!")
+        self.aqua = AquaData(aqua_db_path)
 
     def get_user_id(self, luid: str):
         user_id = self.data.card.get_user_id_from_card(access_code=luid)
@@ -202,16 +164,14 @@ class Importer:
         card_id: int,
     ) -> Dict:
         row = row._asdict()
-        if not self.use_mysql:
-            for column in datetime_columns:
-                ts = row[column["name"]]
-                if ts is None or ts == 0:
-                    continue
-
-                # actuall remove the last 3 zeros for the correct timestamp
-                fixed_ts = int(str(ts)[:-3])
-                # save the datetim object in the dict
-                row[column["name"]] = datetime.fromtimestamp(fixed_ts)
+        for column in datetime_columns:
+            ts = row[column["name"]]
+            if ts is None:
+                continue
+            # actuall remove the last 3 zeros for the correct timestamp
+            fixed_ts = int(str(ts)[:-3])
+            # save the datetim object in the dict
+            row[column["name"]] = datetime.fromtimestamp(fixed_ts)
 
         tmp = {}
         for k, v in row.items():
@@ -251,7 +211,7 @@ class Importer:
             f"SELECT * FROM chusan_user_data WHERE id = {aqua_user_id}"
         )
 
-        # could never be None undless something is really fucked up
+        # could never be None undless somethign is really fucked up
         user_data = None
         for user in user_result:
             user_data = user._asdict()
@@ -455,7 +415,7 @@ class Importer:
             f"SELECT * FROM ongeki_user_data WHERE id = {aqua_user_id}"
         )
 
-        # could never be None undless something is really fucked up
+        # could never be None undless somethign is really fucked up
         user_data = None
         for user in user_result:
             user_data = user._asdict()
@@ -737,396 +697,6 @@ class Importer:
 
             self.logger.info(f"Imported {version_str} userTechCount: {tmp['levelId']}")
 
-    def get_mai2_card_id_by_aqua_row(self, row: Row, user_id_column: str = "user_id"):
-        aqua_user_id = row._asdict()[user_id_column]
-        user_result = self.aqua.execute(
-            f"SELECT * FROM maimai2_user_detail WHERE id = {aqua_user_id}"
-        )
-
-        # could never be None undless something is really fucked up
-        user_data = None
-        for user in user_result:
-            user_data = user._asdict()
-
-        card_id = user_data["aime_card_id"]
-
-        return card_id
-
-    def get_mai2_rating_lists_by_aqua_row(self, row: Row, user_id_column: str = "id"):
-        aqua_user_id = row._asdict()[user_id_column]
-        user_general_data_result = self.aqua.execute(
-            f"SELECT * FROM maimai2_user_general_data WHERE user_id = {aqua_user_id}"
-        )
-
-        ratings = {}
-        for row in user_general_data_result:
-            row = row._asdict()
-            propery_key = row["property_key"]
-            property_value: str = row["property_value"]
-
-            ratings[propery_key] = []
-            if property_value == "":
-                continue
-
-            for rating_str in property_value.split(","):
-                (
-                    music_id_str,
-                    level_str,
-                    romVersion_str,
-                    achievement_str,
-                ) = rating_str.split(":")
-                ratings[propery_key].append(
-                    {
-                        "musicId": int(music_id_str),
-                        "level": int(level_str),
-                        "romVersion": int(romVersion_str),
-                        "achievement": int(achievement_str),
-                    }
-                )
-
-        user_udemae_result = self.aqua.execute(
-            f"SELECT * FROM maimai2_user_udemae WHERE user_id = {aqua_user_id}"
-        )
-        for user_udeame_row in user_udemae_result:
-            user_udeame = user_udeame_row._asdict()
-
-        user_udeame.pop("id")
-        user_udeame.pop("user_id")
-
-        udemae = {inflection.camelize(k, False): v for k, v in user_udeame.items()}
-
-        return (
-            ratings["recent_rating"],
-            ratings["recent_rating_new"],
-            ratings["recent_rating_next"],
-            ratings["recent_rating_next_new"],
-            udemae,
-        )
-
-    def import_mai2(self):
-        game_cfg = Mai2Config()
-        game_cfg.update(yaml.safe_load(open(f"{self.config_folder}/mai2.yaml")))
-
-        base = Mai2Festival(self.config, game_cfg)
-        version_str = Mai2Constants.game_ver_to_string(base.version)
-
-        answer = input(
-            f"Do you really want to import ALL {version_str} data into ARTEMiS? [N/y]: "
-        )
-        if answer.lower() != "y":
-            self.logger.info("User aborted operation")
-            return
-
-        # maimai2_user_detail -> userData
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_detail")
-        for row in result:
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id", "aimeCardId"],
-                card_id=row._asdict()["aime_card_id"],
-            )
-
-            # useless but required
-            tmp["accessCode"] = ""
-            # camel case conversion fix
-            tmp["lastSelectEMoney"] = tmp.pop("lastSelectemoney")
-
-            # convert charaSlot and charaLockSlot
-            # "0;0;0;0;0" to [0, 0, 0, 0, 0]
-            tmp["charaSlot"] = [int(x) for x in tmp["charaSlot"].split(";")]
-            tmp["charaLockSlot"] = [int(x) for x in tmp["charaLockSlot"].split(";")]
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userData": [tmp]}}
-            )
-
-            self.logger.info(f"Imported {version_str} userData: {tmp['user']}")
-
-            # ! Here we insert user rating list
-
-            rating = tmp["playerRating"]
-            (
-                rating_list,
-                new_rating_list,
-                next_rating_list,
-                next_new_rating_list,
-                udemae,
-            ) = self.get_mai2_rating_lists_by_aqua_row(row)
-
-            base.handle_upsert_user_all_api_request(
-                {
-                    "userId": tmp["user"],
-                    "upsertUserAll": {
-                        "userRatingList": [
-                            {
-                                "rating": rating,
-                                "ratingList": rating_list,
-                                "newRatingList": new_rating_list,
-                                "nextRatingList": next_rating_list,
-                                "nextNewRatingList": next_new_rating_list,
-                                "udemae": udemae,
-                            }
-                        ]
-                    },
-                }
-            )
-
-        # maimai2_user_playlog -> userPlaylogList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_playlog")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row, datetime_columns, unused_columns=[], card_id=user
-            )
-            tmp["userId"] = tmp["user"]
-            id_ = tmp.pop("id")
-
-            base.handle_upload_user_playlog_api_request(
-                {"userId": tmp["user"], "userPlaylog": tmp}
-            )
-
-            self.logger.info(f"Imported {version_str} userPlaylog: {id_}")
-
-        # maimai2_user_extend -> userExtend
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_extend")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row, datetime_columns, unused_columns=["id"], card_id=user
-            )
-
-            # convert the str to a list, so it matches the JSON schema
-            tmp["selectedCardList"] = []
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userExtend": [tmp]}}
-            )
-
-            self.logger.info(f"Imported {version_str} userExtend: {tmp['user']}")
-
-        # skip userGhost
-
-        # maimai2_user_option -> userOption
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_option")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row, datetime_columns, unused_columns=["id"], card_id=user
-            )
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userOption": [tmp]}}
-            )
-
-            self.logger.info(f"Imported {version_str} userOption: {tmp['user']}")
-
-        # maimai2_user_activity -> userActivityList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_activity")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                # we do need the id column cuz aqua always sets activityId to 1
-                # and in artemis, activityId will be set to the value of id and id will be dropped
-                unused_columns=[],
-                card_id=user,
-            )
-
-            # using raw operation because wtf
-            base.data.profile.put_profile_activity(tmp["user"], tmp)
-
-            self.logger.info(
-                f"Imported {version_str} userActivity: {tmp['user']}, {tmp['activityId']}"
-            )
-
-        # maimai2_user_charge -> userChargeList
-        # untested
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_charge")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userChargeList": [tmp]}}
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userCharge: {tmp['user']}, {tmp['chargeId']}"
-            )
-
-        # maimai2_user_character -> userCharacterList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_character")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            tmp["point"] = 0
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userCharacterList": [tmp]}}
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userCharacter: {tmp['user']}, {tmp['characterId']}"
-            )
-
-        # maimai2_user_item -> userItemList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_item")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userItemList": [tmp]}}
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userItem: {tmp['user']}, {tmp['itemKind']},{tmp['itemId']}"
-            )
-
-        # maimai2_user_login_bonus -> userLoginBonusList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_login_bonus")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userLoginBonusList": [tmp]}}
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userLoginBonus: {tmp['user']}, {tmp['bonusId']}"
-            )
-
-        # maimai2_user_map -> userMapList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_map")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userMapList": [tmp]}}
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userMap: {tmp['user']}, {tmp['mapId']}"
-            )
-
-        # maimai2_User_music_detail -> userMusicDetailList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_music_detail")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userMusicDetailList": [tmp]}}
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userMusicDetail: {tmp['user']}, {tmp['musicId']}"
-            )
-
-        # maimai2_user_course -> userCourseList
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_course")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userCourseList": [tmp]}}
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userCourse: {tmp['user']}, {tmp['courseId']}"
-            )
-
-        # maimai2_user_favorite -> userFavoriteList
-        # untested
-        result, datetime_columns = self.parse_aqua_db("maimai2_user_favorite")
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-
-            tmp = {
-                "user": tmp["user"],
-                "kind": tmp["itemKind"],
-                "itemIdList": tmp["itemIdList"],
-            }
-
-            base.handle_upsert_user_all_api_request(
-                {"userId": tmp["user"], "upsertUserAll": {"userFavoriteList": [tmp]}}
-            )
-
-            self.logger.info(f"Imported {version_str} userFavorite: {tmp['user']}")
-
-        # maimai2_user_friend_season_ranking -> userFriendSeasonRankingList
-        result, datetime_columns = self.parse_aqua_db(
-            "maimai2_user_friend_season_ranking"
-        )
-        for row in result:
-            user = self.get_mai2_card_id_by_aqua_row(row)
-            tmp = self.parse_aqua_row(
-                row,
-                datetime_columns,
-                unused_columns=["id"],
-                card_id=user,
-            )
-            # user is redundant
-            artemis_user = tmp.pop("user")
-
-            base.handle_upsert_user_all_api_request(
-                {
-                    "userId": artemis_user,
-                    "upsertUserAll": {"userFriendSeasonRankingList": [tmp]},
-                }
-            )
-
-            self.logger.info(
-                f"Imported {version_str} userFriendSeasonRanking: {artemis_user}, {tmp['seasonId']}"
-            )
-
 
 def main():
     parser = argparse.ArgumentParser(description="AQUA to ARTEMiS")
@@ -1136,7 +706,7 @@ def main():
     parser.add_argument(
         "aqua_folder_path",
         type=str,
-        help="Absolute folder path to AQUA folder, where application.properties is located in.",
+        help="Absolute folder path to AQUA /data folder, where db.sqlite is located in",
     )
     args = parser.parse_args()
 
@@ -1147,7 +717,6 @@ def main():
 
     importer.import_chuni()
     importer.import_ongeki()
-    importer.import_mai2()
 
 
 if __name__ == "__main__":