diff --git a/changelog.md b/changelog.md index e1b5642..40d2e48 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,56 @@ # Changelog Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to. +## 20230716 +### General ++ Docker files added (#19) ++ Added support for threading + + This comes with the caviat that enabling it will not allow you to use Ctrl + C to stop the server. + +### Webui ++ Small improvements ++ Add card display + +### Allnet ++ Billing format validation ++ Fix naomitest.html endpoint ++ Add event logging for auths and billing ++ LoaderStateRecorder endpoint handler added + +### Mucha ++ Fixed log level always being "Info" ++ Add stub handler for DownloadState + +### Sword Art Online ++ Support added + +### Crossbeats ++ Added threading to profile loading + + This should cause a noticeable speed-up + +### Card Maker ++ DX Passes fixed ++ Various improvements + +### Diva ++ Added clear status calculation ++ Various minor fixes and improvements + +### Maimai ++ Added support for memorial photo uploads ++ Added support for the following versions + + Festival + + FiNALE ++ Various bug fixes and improvements + +### Wacca ++ Fixed an error that sometimes occoured when trying to unlock songs (#22) + +### Pokken ++ Profile saving added (loading TBA) ++ Use external STUN server for matching by default + + Matching still not working + ## 2023042300 ### Wacca + Time free now works properly diff --git a/core/aimedb.py b/core/aimedb.py index 64bac8d..39d6373 100644 --- a/core/aimedb.py +++ b/core/aimedb.py @@ -63,7 +63,7 @@ class AimedbProtocol(Protocol): try: decrypted = cipher.decrypt(data) - except: + except Exception: self.logger.error(f"Failed to decrypt {data.hex()}") return None diff --git a/core/data/schema/base.py b/core/data/schema/base.py index 319101f..a53392f 100644 --- a/core/data/schema/base.py +++ b/core/data/schema/base.py @@ -58,7 +58,7 @@ class BaseData: self.logger.error(f"UnicodeEncodeError error {e}") return None - except: + except Exception: try: res = self.conn.execute(sql, opts) @@ -70,7 +70,7 @@ class BaseData: self.logger.error(f"UnicodeEncodeError error {e}") return None - except: + except Exception: self.logger.error(f"Unknown error") raise diff --git a/core/frontend.py b/core/frontend.py index 9eb30e6..f01be50 100644 --- a/core/frontend.py +++ b/core/frontend.py @@ -130,7 +130,7 @@ class FE_Gate(FE_Base): if b"e" in request.args: try: err = int(request.args[b"e"][0].decode()) - except: + except Exception: err = 0 else: diff --git a/core/mucha.py b/core/mucha.py index 6bb34d7..8d9dd8e 100644 --- a/core/mucha.py +++ b/core/mucha.py @@ -68,10 +68,13 @@ class MuchaServlet: return b"RESULTS=000" # TODO: Decrypt S/N + b_key = b"" + for x in range(8): + b_key += req.sendDate[(x - 1) & 7].encode() - #cipher = Blowfish.new(req.sendDate.encode(), Blowfish.MODE_ECB) - #sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum)) - #self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}") + cipher = Blowfish.new(b_key, Blowfish.MODE_ECB) + sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum)) + self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}") resp = MuchaAuthResponse( f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}" @@ -131,7 +134,7 @@ class MuchaServlet: return ret - except: + except Exception: self.logger.error(f"Error processing mucha request {data}") return None @@ -143,7 +146,7 @@ class MuchaServlet: return urlencode.encode() - except: + except Exception: self.logger.error("Error processing mucha response") return None diff --git a/example_config/mai2.yaml b/example_config/mai2.yaml index d89f5d7..8557151 100644 --- a/example_config/mai2.yaml +++ b/example_config/mai2.yaml @@ -5,4 +5,10 @@ server: deliver: enable: False udbdl_enable: False - content_folder: "" \ No newline at end of file + content_folder: "" + +uploads: + photos: False + photos_dir: "" + movies: False + movies_dir: "" diff --git a/requirements.txt b/requirements.txt index 6d37728..53d867a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ bcrypt jinja2 protobuf autobahn +pillow diff --git a/titles/chuni/base.py b/titles/chuni/base.py index ed8d0fb..4c4361c 100644 --- a/titles/chuni/base.py +++ b/titles/chuni/base.py @@ -644,7 +644,7 @@ class ChuniBase: upsert["userData"][0]["userName"] = self.read_wtf8( upsert["userData"][0]["userName"] ) - except: + except Exception: pass self.data.profile.put_profile_data( diff --git a/titles/cxb/base.py b/titles/cxb/base.py index a649f31..89e9cc3 100644 --- a/titles/cxb/base.py +++ b/titles/cxb/base.py @@ -197,7 +197,7 @@ class CxbBase: v_profile = self.data.profile.get_profile_index(0, uid, self.version) v_profile_data = v_profile["data"] versionindex.append(int(v_profile_data["appVersion"])) - except: + except Exception: versionindex.append("10400") def handle_action_loadrange_request(self, data: Dict) -> Dict: @@ -286,7 +286,7 @@ class CxbBase: # REV Omnimix Version Fetcher gameversion = data["saveindex"]["data"][0][2] self.logger.warning(f"Game Version is {gameversion}") - except: + except Exception: pass if "10205" in gameversion: @@ -348,7 +348,7 @@ class CxbBase: # Sunrise try: profileIndex = save_data["index"].index("0") - except: + except Exception: return {"data": ""} # Maybe profile = json.loads(save_data["data"][profileIndex]) @@ -496,7 +496,7 @@ class CxbBase: score=int(rid["sc"][0]), clear=rid["clear"], ) - except: + except Exception: self.data.score.put_ranking( user_id=uid, rev_id=int(rid["rid"]), @@ -514,7 +514,7 @@ class CxbBase: score=int(rid["sc"][0]), clear=0, ) - except: + except Exception: self.data.score.put_ranking( user_id=uid, rev_id=int(rid["rid"]), diff --git a/titles/cxb/read.py b/titles/cxb/read.py index cf2d8e1..06a171f 100644 --- a/titles/cxb/read.py +++ b/titles/cxb/read.py @@ -123,5 +123,5 @@ class CxbReader(BaseReader): genre, int(row["easy"].replace("Easy ", "").replace("N/A", "0")), ) - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") diff --git a/titles/mai2/base.py b/titles/mai2/base.py index ad9aafd..f1b04b6 100644 --- a/titles/mai2/base.py +++ b/titles/mai2/base.py @@ -1,6 +1,9 @@ -from datetime import datetime, date, timedelta +from datetime import datetime from typing import Any, Dict, List import logging +from base64 import b64decode +from os import path, stat, remove +from PIL import ImageFile from core.config import CoreConfig from titles.mai2.const import Mai2Constants @@ -89,7 +92,7 @@ class Mai2Base: for i, charge in enumerate(game_charge_list): charge_list.append( { - "orderId": i, + "orderId": i + 1, "chargeId": charge["ticketId"], "price": charge["price"], "startDate": "2017-12-05 07:00:00.0", @@ -773,4 +776,89 @@ class Mai2Base: self.logger.debug(data) def handle_upload_user_photo_api_request(self, data: Dict) -> Dict: - self.logger.debug(data) \ No newline at end of file + if not self.game_config.uploads.photos or not self.game_config.uploads.photos_dir: + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + photo = data.get("userPhoto", {}) + + if photo is None or not photo: + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + order_id = int(photo.get("orderId", -1)) + user_id = int(photo.get("userId", -1)) + div_num = int(photo.get("divNumber", -1)) + div_len = int(photo.get("divLength", -1)) + div_data = photo.get("divData", "") + playlog_id = int(photo.get("playlogId", -1)) + track_num = int(photo.get("trackNo", -1)) + upload_date = photo.get("uploadDate", "") + + if order_id < 0 or user_id <= 0 or div_num < 0 or div_len <= 0 or not div_data or playlog_id < 0 or track_num <= 0 or not upload_date: + self.logger.warn(f"Malformed photo upload request") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + if order_id == 0 and div_num > 0: + self.logger.warn(f"Failed to set orderId properly (still 0 after first chunk)") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + if div_num == 0 and order_id > 0: + self.logger.warn(f"First chuck re-send, Ignore") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + if div_num >= div_len: + self.logger.warn(f"Sent extra chunks ({div_num} >= {div_len})") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + if div_len >= 100: + self.logger.warn(f"Photo too large ({div_len} * 10240 = {div_len * 10240} bytes)") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + ret_code = order_id + 1 + photo_chunk = b64decode(div_data) + + if len(photo_chunk) > 10240 or (len(photo_chunk) < 10240 and div_num + 1 != div_len): + self.logger.warn(f"Incorrect data size after decoding (Expected 10240, got {len(photo_chunk)})") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + out_name = f"{self.game_config.uploads.photos_dir}/{user_id}_{playlog_id}_{track_num}" + + if not path.exists(f"{out_name}.bin") and div_num != 0: + self.logger.warn(f"Out of order photo upload (div_num {div_num})") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + if path.exists(f"{out_name}.bin") and div_num == 0: + self.logger.warn(f"Duplicate file upload") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + elif path.exists(f"{out_name}.bin"): + fstats = stat(f"{out_name}.bin") + if fstats.st_size != 10240 * div_num: + self.logger.warn(f"Out of order photo upload (trying to upload div {div_num}, expected div {fstats.st_size / 10240} for file sized {fstats.st_size} bytes)") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + try: + with open(f"{out_name}.bin", "ab") as f: + f.write(photo_chunk) + + except Exception: + self.logger.error(f"Failed writing to {out_name}.bin") + return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'} + + if div_num + 1 == div_len and path.exists(f"{out_name}.bin"): + try: + p = ImageFile.Parser() + with open(f"{out_name}.bin", "rb") as f: + p.feed(f.read()) + + im = p.close() + im.save(f"{out_name}.jpeg") + except Exception: + self.logger.error(f"File {out_name}.bin failed image validation") + + try: + remove(f"{out_name}.bin") + + except Exception: + self.logger.error(f"Failed to delete {out_name}.bin, please remove it manually") + + return {'returnCode': ret_code, 'apiName': 'UploadUserPhotoApi'} diff --git a/titles/mai2/dx.py b/titles/mai2/dx.py index 185dda0..fba092a 100644 --- a/titles/mai2/dx.py +++ b/titles/mai2/dx.py @@ -117,7 +117,7 @@ class Mai2DX(Mai2Base): if "userGhost" in upsert: for ghost in upsert["userGhost"]: - self.data.profile.put_profile_extend(user_id, self.version, ghost) + self.data.profile.put_profile_ghost(user_id, self.version, ghost) if "userOption" in upsert and len(upsert["userOption"]) > 0: self.data.profile.put_profile_option( @@ -217,9 +217,6 @@ class Mai2DX(Mai2Base): return {"returnCode": 1, "apiName": "UpsertUserAllApi"} - def handle_user_logout_api_request(self, data: Dict) -> Dict: - return {"returnCode": 1} - def handle_get_user_data_api_request(self, data: Dict) -> Dict: profile = self.data.profile.get_profile_detail(data["userId"], self.version) if profile is None: @@ -568,3 +565,10 @@ class Mai2DX(Mai2Base): "nextIndex": next_index, "userMusicList": [{"userMusicDetailList": music_detail_list}], } + + def handle_user_login_api_request(self, data: Dict) -> Dict: + ret = super().handle_user_login_api_request(data) + if ret is None or not ret: + return ret + ret['loginId'] = ret.get('loginCount', 0) + return ret diff --git a/titles/mai2/index.py b/titles/mai2/index.py index c250894..9652fc8 100644 --- a/titles/mai2/index.py +++ b/titles/mai2/index.py @@ -7,7 +7,7 @@ import string import logging, coloredlogs import zlib from logging.handlers import TimedRotatingFileHandler -from os import path +from os import path, mkdir from typing import Tuple from core.config import CoreConfig @@ -109,6 +109,19 @@ class Mai2Servlet: f"{core_cfg.title.hostname}", ) + def setup(self): + if self.game_cfg.uploads.photos and self.game_cfg.uploads.photos_dir and not path.exists(self.game_cfg.uploads.photos_dir): + try: + mkdir(self.game_cfg.uploads.photos_dir) + except Exception: + self.logger.error(f"Failed to make photo upload directory at {self.game_cfg.uploads.photos_dir}") + + if self.game_cfg.uploads.movies and self.game_cfg.uploads.movies_dir and not path.exists(self.game_cfg.uploads.movies_dir): + try: + mkdir(self.game_cfg.uploads.movies_dir) + except Exception: + self.logger.error(f"Failed to make movie upload directory at {self.game_cfg.uploads.movies_dir}") + def render_POST(self, request: Request, version: int, url_path: str) -> bytes: if url_path.lower() == "ping": return zlib.compress(b'{"returnCode": "1"}') diff --git a/titles/sao/read.py b/titles/sao/read.py index d70c275..649fa02 100644 --- a/titles/sao/read.py +++ b/titles/sao/read.py @@ -65,7 +65,7 @@ class SaoReader(BaseReader): ) except Exception as err: print(err) - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") self.logger.info("Now reading HeroLog.csv") @@ -99,7 +99,7 @@ class SaoReader(BaseReader): ) except Exception as err: print(err) - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") self.logger.info("Now reading Equipment.csv") @@ -131,7 +131,7 @@ class SaoReader(BaseReader): ) except Exception as err: print(err) - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") self.logger.info("Now reading Item.csv") @@ -161,7 +161,7 @@ class SaoReader(BaseReader): ) except Exception as err: print(err) - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") self.logger.info("Now reading SupportLog.csv") @@ -193,7 +193,7 @@ class SaoReader(BaseReader): ) except Exception as err: print(err) - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") self.logger.info("Now reading Title.csv") @@ -226,7 +226,7 @@ class SaoReader(BaseReader): print(err) elif len(titleId) < 6: # current server code cannot have multiple lengths for the id continue - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") self.logger.info("Now reading RareDropTable.csv") @@ -250,5 +250,5 @@ class SaoReader(BaseReader): ) except Exception as err: print(err) - except: + except Exception: self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping") diff --git a/titles/wacca/base.py b/titles/wacca/base.py index cca6aa5..2351001 100644 --- a/titles/wacca/base.py +++ b/titles/wacca/base.py @@ -426,7 +426,7 @@ class WaccaBase: elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]: resp.userItems.noteSounds.append(itm_send) - except: + except Exception: self.logger.error( f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}" ) diff --git a/titles/wacca/const.py b/titles/wacca/const.py index 284d236..b25d3ac 100644 --- a/titles/wacca/const.py +++ b/titles/wacca/const.py @@ -221,5 +221,5 @@ class WaccaConstants: cls.Region.YAMANASHI, cls.Region.WAKAYAMA, ][region] - except: + except Exception: return None diff --git a/titles/wacca/index.py b/titles/wacca/index.py index a59cda1..e36c295 100644 --- a/titles/wacca/index.py +++ b/titles/wacca/index.py @@ -93,7 +93,7 @@ class WaccaServlet: try: req_json = json.loads(request.content.getvalue()) version_full = Version(req_json["appVersion"]) - except: + except Exception: self.logger.error( f"Failed to parse request to {url_path} -> {request.content.getvalue()}" ) diff --git a/titles/wacca/lily.py b/titles/wacca/lily.py index 6ac60de..3ac03fa 100644 --- a/titles/wacca/lily.py +++ b/titles/wacca/lily.py @@ -424,7 +424,7 @@ class WaccaLily(WaccaS): elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]: resp.userItems.noteSounds.append(itm_send) - except: + except Exception: self.logger.error( f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}" ) diff --git a/titles/wacca/reverse.py b/titles/wacca/reverse.py index 1711013..728ef0a 100644 --- a/titles/wacca/reverse.py +++ b/titles/wacca/reverse.py @@ -289,7 +289,7 @@ class WaccaReverse(WaccaLilyR): elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]: resp.userItems.noteSounds.append(itm_send) - except: + except Exception: self.logger.error( f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}" )