Merge remote-tracking branch 'origin/develop' into fork_develop

changelog.md

@@ -1,6 +1,56 @@
 # Changelog
 Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
 
+## 20230716
+### General
++ Docker files added (#19)
++ Added support for threading
+  + This comes with the caviat that enabling it will not allow you to use Ctrl + C to stop the server.
+
+### Webui
++ Small improvements
++ Add card display
+
+### Allnet
++ Billing format validation
++ Fix naomitest.html endpoint
++ Add event logging for auths and billing
++ LoaderStateRecorder endpoint handler added
+
+### Mucha
++ Fixed log level always being "Info"
++ Add stub handler for DownloadState
+
+### Sword Art Online
++ Support added
+
+### Crossbeats
++ Added threading to profile loading
+  + This should cause a noticeable speed-up
+
+### Card Maker
++ DX Passes fixed
++ Various improvements
+
+### Diva
++ Added clear status calculation
++ Various minor fixes and improvements
+
+### Maimai
++ Added support for memorial photo uploads
++ Added support for the following versions
+  + Festival
+  + FiNALE
++ Various bug fixes and improvements
+
+### Wacca
++ Fixed an error that sometimes occoured when trying to unlock songs (#22)
+
+### Pokken
++ Profile saving added (loading TBA)
++ Use external STUN server for matching by default
+  + Matching still not working
+
 ## 2023042300
 ### Wacca
 + Time free now works properly

core/aimedb.py

@@ -63,7 +63,7 @@ class AimedbProtocol(Protocol):
 
         try:
             decrypted = cipher.decrypt(data)
-        except:
+        except Exception:
             self.logger.error(f"Failed to decrypt {data.hex()}")
             return None
 

core/data/schema/base.py

@@ -58,7 +58,7 @@ class BaseData:
             self.logger.error(f"UnicodeEncodeError error {e}")
             return None
 
-        except:
+        except Exception:
             try:
                 res = self.conn.execute(sql, opts)
 
@@ -70,7 +70,7 @@ class BaseData:
                 self.logger.error(f"UnicodeEncodeError error {e}")
                 return None
 
-            except:
+            except Exception:
                 self.logger.error(f"Unknown error")
                 raise
 

core/frontend.py

@@ -130,7 +130,7 @@ class FE_Gate(FE_Base):
         if b"e" in request.args:
             try:
                 err = int(request.args[b"e"][0].decode())
-            except:
+            except Exception:
                 err = 0
 
         else:

core/mucha.py

@@ -68,10 +68,13 @@ class MuchaServlet:
             return b"RESULTS=000"
 
         # TODO: Decrypt S/N
+        b_key = b""
+        for x in range(8):
+            b_key += req.sendDate[(x - 1) & 7].encode()
 
-        #cipher = Blowfish.new(req.sendDate.encode(), Blowfish.MODE_ECB)
+        cipher = Blowfish.new(b_key, Blowfish.MODE_ECB)
-        #sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))
+        sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))
-        #self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}")
+        self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}")
 
         resp = MuchaAuthResponse(
             f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}"
@@ -131,7 +134,7 @@ class MuchaServlet:
 
             return ret
 
-        except:
+        except Exception:
             self.logger.error(f"Error processing mucha request {data}")
             return None
 
@@ -143,7 +146,7 @@ class MuchaServlet:
 
             return urlencode.encode()
 
-        except:
+        except Exception:
             self.logger.error("Error processing mucha response")
             return None
 

example_config/mai2.yaml

@@ -5,4 +5,10 @@ server:
 deliver:
   enable: False
   udbdl_enable: False
-  content_folder: ""
+  content_folder: ""
+
+uploads:
+  photos: False
+  photos_dir: ""
+  movies: False
+  movies_dir: ""

requirements.txt

@@ -17,3 +17,4 @@ bcrypt
 jinja2
 protobuf
 autobahn
+pillow

titles/chuni/base.py

@@ -644,7 +644,7 @@ class ChuniBase:
                 upsert["userData"][0]["userName"] = self.read_wtf8(
                     upsert["userData"][0]["userName"]
                 )
-            except:
+            except Exception:
                 pass
 
             self.data.profile.put_profile_data(

titles/cxb/base.py

@@ -197,7 +197,7 @@ class CxbBase:
             v_profile = self.data.profile.get_profile_index(0, uid, self.version)
             v_profile_data = v_profile["data"]
             versionindex.append(int(v_profile_data["appVersion"]))
-        except:
+        except Exception:
             versionindex.append("10400")
 
     def handle_action_loadrange_request(self, data: Dict) -> Dict:
@@ -286,7 +286,7 @@ class CxbBase:
             # REV Omnimix Version Fetcher
             gameversion = data["saveindex"]["data"][0][2]
             self.logger.warning(f"Game Version is {gameversion}")
-        except:
+        except Exception:
             pass
 
         if "10205" in gameversion:
@@ -348,7 +348,7 @@ class CxbBase:
         # Sunrise
         try:
             profileIndex = save_data["index"].index("0")
-        except:
+        except Exception:
             return {"data": ""}  # Maybe
 
         profile = json.loads(save_data["data"][profileIndex])
@@ -496,7 +496,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=rid["clear"],
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
@@ -514,7 +514,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=0,
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),

titles/cxb/read.py

@@ -123,5 +123,5 @@ class CxbReader(BaseReader):
                             genre,
                             int(row["easy"].replace("Easy ", "").replace("N/A", "0")),
                         )
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")

titles/mai2/base.py

@@ -1,6 +1,9 @@
-from datetime import datetime, date, timedelta
+from datetime import datetime
 from typing import Any, Dict, List
 import logging
+from base64 import b64decode
+from os import path, stat, remove
+from PIL import ImageFile
 
 from core.config import CoreConfig
 from titles.mai2.const import Mai2Constants
@@ -89,7 +92,7 @@ class Mai2Base:
         for i, charge in enumerate(game_charge_list):
             charge_list.append(
                 {
-                    "orderId": i,
+                    "orderId": i + 1,
                     "chargeId": charge["ticketId"],
                     "price": charge["price"],
                     "startDate": "2017-12-05 07:00:00.0",
@@ -773,4 +776,89 @@ class Mai2Base:
         self.logger.debug(data)
 
     def handle_upload_user_photo_api_request(self, data: Dict) -> Dict:
-        self.logger.debug(data)
+        if not self.game_config.uploads.photos or not self.game_config.uploads.photos_dir:
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        photo = data.get("userPhoto", {})
+
+        if photo is None or not photo:
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        order_id = int(photo.get("orderId", -1))
+        user_id = int(photo.get("userId", -1))
+        div_num = int(photo.get("divNumber", -1))
+        div_len = int(photo.get("divLength", -1))
+        div_data = photo.get("divData", "")
+        playlog_id = int(photo.get("playlogId", -1))
+        track_num = int(photo.get("trackNo", -1))
+        upload_date = photo.get("uploadDate", "")
+
+        if order_id < 0 or user_id <= 0 or div_num < 0 or div_len <= 0 or not div_data or playlog_id < 0 or track_num <= 0 or not upload_date:
+            self.logger.warn(f"Malformed photo upload request")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        if order_id == 0 and div_num > 0:
+            self.logger.warn(f"Failed to set orderId properly (still 0 after first chunk)")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        if div_num == 0 and order_id > 0:
+            self.logger.warn(f"First chuck re-send, Ignore")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        if div_num >= div_len:
+            self.logger.warn(f"Sent extra chunks ({div_num} >= {div_len})")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        if div_len >= 100:
+            self.logger.warn(f"Photo too large ({div_len} * 10240 = {div_len * 10240} bytes)")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        ret_code = order_id + 1
+        photo_chunk = b64decode(div_data)
+
+        if len(photo_chunk) > 10240 or (len(photo_chunk) < 10240 and div_num + 1 != div_len):
+            self.logger.warn(f"Incorrect data size after decoding (Expected 10240, got {len(photo_chunk)})")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        out_name = f"{self.game_config.uploads.photos_dir}/{user_id}_{playlog_id}_{track_num}"
+
+        if not path.exists(f"{out_name}.bin") and div_num != 0:
+            self.logger.warn(f"Out of order photo upload (div_num {div_num})")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+    
+        if path.exists(f"{out_name}.bin") and div_num == 0:
+            self.logger.warn(f"Duplicate file upload")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        elif path.exists(f"{out_name}.bin"):
+            fstats = stat(f"{out_name}.bin")
+            if fstats.st_size != 10240 * div_num:
+                self.logger.warn(f"Out of order photo upload (trying to upload div {div_num}, expected div {fstats.st_size / 10240} for file sized {fstats.st_size} bytes)")
+                return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        try:
+            with open(f"{out_name}.bin", "ab") as f:
+                f.write(photo_chunk)
+        
+        except Exception:
+            self.logger.error(f"Failed writing to {out_name}.bin")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        if div_num + 1 == div_len and path.exists(f"{out_name}.bin"):
+            try:
+                p = ImageFile.Parser()
+                with open(f"{out_name}.bin", "rb") as f:
+                    p.feed(f.read())
+                
+                im = p.close()
+                im.save(f"{out_name}.jpeg")
+            except Exception:
+                self.logger.error(f"File {out_name}.bin failed image validation")
+            
+            try:
+                remove(f"{out_name}.bin")
+            
+            except Exception:
+                self.logger.error(f"Failed to delete {out_name}.bin, please remove it manually")
+
+        return {'returnCode': ret_code, 'apiName': 'UploadUserPhotoApi'}

titles/mai2/dx.py

@@ -117,7 +117,7 @@ class Mai2DX(Mai2Base):
 
         if "userGhost" in upsert:
             for ghost in upsert["userGhost"]:
-                self.data.profile.put_profile_extend(user_id, self.version, ghost)
+                self.data.profile.put_profile_ghost(user_id, self.version, ghost)
 
         if "userOption" in upsert and len(upsert["userOption"]) > 0:
             self.data.profile.put_profile_option(
@@ -217,9 +217,6 @@ class Mai2DX(Mai2Base):
 
         return {"returnCode": 1, "apiName": "UpsertUserAllApi"}
 
-    def handle_user_logout_api_request(self, data: Dict) -> Dict:
-        return {"returnCode": 1}
-
     def handle_get_user_data_api_request(self, data: Dict) -> Dict:
         profile = self.data.profile.get_profile_detail(data["userId"], self.version)
         if profile is None:
@@ -568,3 +565,10 @@ class Mai2DX(Mai2Base):
             "nextIndex": next_index,
             "userMusicList": [{"userMusicDetailList": music_detail_list}],
         }
+
+    def handle_user_login_api_request(self, data: Dict) -> Dict:
+        ret = super().handle_user_login_api_request(data)
+        if ret is None or not ret:
+            return ret
+        ret['loginId'] = ret.get('loginCount', 0)
+        return ret

titles/mai2/index.py

@@ -7,7 +7,7 @@ import string
 import logging, coloredlogs
 import zlib
 from logging.handlers import TimedRotatingFileHandler
-from os import path
+from os import path, mkdir
 from typing import Tuple
 
 from core.config import CoreConfig
@@ -109,6 +109,19 @@ class Mai2Servlet:
             f"{core_cfg.title.hostname}",
         )
 
+    def setup(self):
+        if self.game_cfg.uploads.photos and self.game_cfg.uploads.photos_dir and not path.exists(self.game_cfg.uploads.photos_dir):
+            try:
+                mkdir(self.game_cfg.uploads.photos_dir)
+            except Exception:
+                self.logger.error(f"Failed to make photo upload directory at {self.game_cfg.uploads.photos_dir}")
+
+        if self.game_cfg.uploads.movies and self.game_cfg.uploads.movies_dir and not path.exists(self.game_cfg.uploads.movies_dir):
+            try:
+                mkdir(self.game_cfg.uploads.movies_dir)
+            except Exception:
+                self.logger.error(f"Failed to make movie upload directory at {self.game_cfg.uploads.movies_dir}")
+
     def render_POST(self, request: Request, version: int, url_path: str) -> bytes:
         if url_path.lower() == "ping":
             return zlib.compress(b'{"returnCode": "1"}')

titles/sao/read.py

@@ -65,7 +65,7 @@ class SaoReader(BaseReader):
                         )
                     except Exception as err:
                         print(err)
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
         
         self.logger.info("Now reading HeroLog.csv")
@@ -99,7 +99,7 @@ class SaoReader(BaseReader):
                         )
                     except Exception as err:
                         print(err)
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
         
         self.logger.info("Now reading Equipment.csv")
@@ -131,7 +131,7 @@ class SaoReader(BaseReader):
                         )
                     except Exception as err:
                         print(err)
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
 
         self.logger.info("Now reading Item.csv")
@@ -161,7 +161,7 @@ class SaoReader(BaseReader):
                         )
                     except Exception as err:
                         print(err)
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
         
         self.logger.info("Now reading SupportLog.csv")
@@ -193,7 +193,7 @@ class SaoReader(BaseReader):
                         )
                     except Exception as err:
                         print(err)
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
             
         self.logger.info("Now reading Title.csv")
@@ -226,7 +226,7 @@ class SaoReader(BaseReader):
                             print(err)
                     elif len(titleId) < 6: # current server code cannot have multiple lengths for the id
                         continue
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
 
         self.logger.info("Now reading RareDropTable.csv")
@@ -250,5 +250,5 @@ class SaoReader(BaseReader):
                         )
                     except Exception as err:
                         print(err)
-        except:
+        except Exception:
             self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")

titles/wacca/base.py

@@ -426,7 +426,7 @@ class WaccaBase:
                         elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]:
                             resp.userItems.noteSounds.append(itm_send)
 
-                except:
+                except Exception:
                     self.logger.error(
                         f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}"
                     )

titles/wacca/const.py

@@ -221,5 +221,5 @@ class WaccaConstants:
                 cls.Region.YAMANASHI,
                 cls.Region.WAKAYAMA,
             ][region]
-        except:
+        except Exception:
             return None

titles/wacca/index.py

@@ -93,7 +93,7 @@ class WaccaServlet:
         try:
             req_json = json.loads(request.content.getvalue())
             version_full = Version(req_json["appVersion"])
-        except:
+        except Exception:
             self.logger.error(
                 f"Failed to parse request to {url_path} -> {request.content.getvalue()}"
             )

titles/wacca/lily.py

@@ -424,7 +424,7 @@ class WaccaLily(WaccaS):
                         elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]:
                             resp.userItems.noteSounds.append(itm_send)
 
-                except:
+                except Exception:
                     self.logger.error(
                         f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}"
                     )

titles/wacca/reverse.py

@@ -289,7 +289,7 @@ class WaccaReverse(WaccaLilyR):
                         elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]:
                             resp.userItems.noteSounds.append(itm_send)
 
-                except:
+                except Exception:
                     self.logger.error(
                         f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}"
                     )