fix: 处理 curseforge 响应错误的情况

HMCL-dev/HMCL#3904

Author: z0z0r4

mcim_sync/config.py

@@ -61,7 +61,7 @@ class ConfigModel(BaseModel):
     interval: JobInterval = JobInterval()
     max_workers: int = 8
     curseforge_chunk_size: int = 1000
-    modrinth_chunk_size: int = 1000
+    modrinth_chunk_size: int = 100
     curseforge_delay: Union[float, int] = 1
     modrinth_delay: Union[float, int] = 1
     curseforge_api_key: str = "<api key>"

mcim_sync/sync/curseforge.py

@@ -116,7 +116,7 @@ def sync_mod_all_files(
 
 def sync_mod_all_files_at_once(
     modId: int, latestFiles: List[dict], need_to_cache: bool = True
-) -> int:
+) -> Optional[int]:
     res = get_mod_files(modId, index=0, pageSize=10000)
 
     append_model_from_files_res(
@@ -127,11 +127,16 @@ def sync_mod_all_files_at_once(
 
     page = Pagination(**res["pagination"])
 
+    if page.resultCount != page.totalCount:
+        log.warning(
+            f"ResultCount {page.resultCount} != TotalCount {page.totalCount} for mod {modId}, response maybe incomplete, passing sync"
+        )
+        return None
     removed_count = mongodb_engine.remove(
         File, File.modId == modId, query.not_in(File.id, file_id_list)
     )
     log.info(
-        f"Finished sync mod {modId}, total {page.totalCount} files, removed {removed_count} files"
+        f"Finished sync mod {modId}, total {page.totalCount} files, resultCount {page.resultCount}, removed {removed_count} files"
     )
 
     return page.totalCount
@@ -144,8 +149,6 @@ def sync_mod(modId: int) -> Optional[ProjectDetail]:
             res = get_mod(modId)
             mod_model = Mod(**res)
             if mod_model.gameId == 432:
-                submitter.add(mod_model)
-
                 # version_count = sync_mod_all_files(
                 #     modId,
                 #     latestFiles=res["latestFiles"],
@@ -158,13 +161,19 @@ def sync_mod(modId: int) -> Optional[ProjectDetail]:
                     need_to_cache=True if res["classId"] == 6 else False,
                 )
 
+                if version_count is None:
+                    return None
+
                 return ProjectDetail(
                     id=res["id"],
                     name=res["name"],
                     version_count=version_count,
                 )
             else:
                 log.debug(f"Mod {modId} gameId is not 432")
+
+            # 最后再添加，以防未成功刷新版本列表而更新 Mod 信息
+            submitter.add(mod_model)
     except ResponseCodeException as e:
         if e.status_code == 404:
             log.error(f"Mod {modId} not found!")
@@ -227,10 +236,12 @@ def sync_categories(
         else:
             raise e
 
+
 class ModsSearchSortField(int, Enum):
     """
     https://docs.curseforge.com/rest-api/#tocS_ModsSearchSortField
     """
+
     Featured = 1
     Popularity = 2
     LastUpdated = 3
@@ -249,6 +260,7 @@ class ModLoaderType(int, Enum):
     """
     https://docs.curseforge.com/rest-api/#tocS_ModLoaderType
     """
+
     Any = 0
     Forge = 1
     Cauldron = 2
@@ -257,13 +269,16 @@ class ModLoaderType(int, Enum):
     Quilt = 5
     NeoForge = 6
 
+
 class ModsSearchSortOrder(str, Enum):
     """
     'asc' if sort is in ascending order, 'desc' if sort is in descending order
     """
+
     ASC = "asc"
     DESC = "desc"
 
+
 @retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
 def fetch_search_result(
     gameId: int = 432,
@@ -282,7 +297,8 @@ def fetch_search_result(
     primaryAuthorId: Optional[int] = None,
     slug: Optional[str] = None,
     index: Optional[int] = None,
-    pageSize: Optional[int] = 50):
+    pageSize: Optional[int] = 50,
+):
     """
     Fetch search result from CurseForge API.
     """
@@ -304,12 +320,12 @@ def fetch_search_result(
             primaryAuthorId=primaryAuthorId,
             slug=slug,
             index=index,
-            pageSize=pageSize
+            pageSize=pageSize,
         )
         return res
     except ResponseCodeException as e:
         if e.status_code == 404:
             log.error("Search result not found!")
             return None
         else:
-            raise e
+            raise e

mcim_sync/tasks/curseforge.py

@@ -55,10 +55,21 @@ def refresh_curseforge_with_modify_date() -> bool:
     else:
         curseforge_pool.shutdown()
 
+    failed_count = len(curseforge_expired_modids) - len(projects_detail_info)
+    failed_modids = [
+        modid for modid in curseforge_expired_modids if modid not in projects_detail_info
+    ]
+    log.info(
+        f"CurseForge expired data sync finished, total: {len(curseforge_expired_modids)}, "
+        f"success: {len(projects_detail_info)}, failed: {failed_count}, "
+        f"failed modids: {failed_modids if failed_modids else 'None'}"
+    )
+
     if config.telegram_bot:
         notification = RefreshNotification(
             platform=Platform.CURSEFORGE,
             projects_detail_info=projects_detail_info,
+            failed_count=failed_count,
         )
         notification.send_to_telegram()
         log.info("CurseForge refresh message sent to telegram.")

mcim_sync/utils/model_submitter/__init__.py

@@ -60,6 +60,11 @@ def close(self) -> None:
         self.flush()
         log.debug(f"ModelSubmitter finished, total submitted: {self.total_submitted}")
 
+    def clear(self) -> None:
+        """清空待保存的模型"""
+        self.models.clear()
+        log.debug("Cleared pending models.")
+
     @property
     def pending_count(self) -> int:
         """待保存的模型数量"""

mcim_sync/utils/telegram/__init__.py

@@ -144,14 +144,23 @@ def send_to_telegram(cls) -> int:
 class RefreshNotification(Notification):
     platform: Platform
     projects_detail_info: List[ProjectDetail]
+    failed_count = 0  # 失败的模组数量
 
-    def __init__(self, platform: Platform, projects_detail_info: List[ProjectDetail]):
+    def __init__(
+        self,
+        platform: Platform,
+        projects_detail_info: List[ProjectDetail],
+        failed_count: Optional[int] = 0,
+    ):
         self.platform = platform
         self.projects_detail_info = projects_detail_info
+        self.failed_count = failed_count if failed_count is not None else 0
 
     def send_to_telegram(self) -> int:
         sync_message = escape_markdown(
-            f"{self.platform.value} 缓存刷新完成，共刷新 {len(self.projects_detail_info)} 个模组",
+            f"{self.platform.value} 缓存刷新完成，共刷新 {len(self.projects_detail_info)} 个模组, {self.failed_count} 个模组刷新失败\n"
+            if self.failed_count > 0
+            else f"{self.platform.value} 缓存刷新完成，共刷新 {len(self.projects_detail_info)} 个模组\n"
         )
         if self.projects_detail_info:
             sync_message += make_project_detail_blockquote(self.projects_detail_info)

start.py

@@ -92,7 +92,7 @@ def main():
                 year="*",
             ),
             name="curseforge_categories_refresh",
-            next_run_time=datetime.datetime.now(),  # 立即执行一次任务
+            # next_run_time=datetime.datetime.now(),  # 立即执行一次任务
         )
 
     if config.job_config.modrinth_tags:
@@ -109,7 +109,7 @@ def main():
                 year="*",
             ),
             name="modrinth_refresh_tags",
-            next_run_time=datetime.datetime.now(),  # 立即执行一次任务
+            # next_run_time=datetime.datetime.now(),  # 立即执行一次任务
         )
 
     if config.telegram_bot and config.job_config.global_statistics:

tests/conftest.py

@@ -4,7 +4,10 @@
 import pytest
 import os
 
-config: dict = json.loads(os.environ.get("CONFIG"))
+if not os.path.exists("config.json"):
+    config: dict = json.loads(os.environ.get("CONFIG"))
+else:
+    config = json.load(open("config.json", "r", encoding="utf-8"))
 headers = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
     "Accept": "application/json",