feat: Modrinth 增加项目存活状态返回并实现项目删除

Author: z0z0r4

mcim_sync/checker/modrinth.py

@@ -1,4 +1,4 @@
-from typing import Union, List, Set
+from typing import Union, List, Set, Tuple
 import datetime
 import time
 
@@ -28,18 +28,27 @@
 MODRINTH_DELAY: Union[float, int] = config.modrinth_delay
 
 
-def check_modrinth_data_updated(projects: List[Project]) -> Set[str]:
+def check_modrinth_data_updated_and_alive(projects: List[Project]) -> tuple[set[str], set[str]]:
     project_info = {
         project.id: {"sync_date": project.updated, "versions": project.versions}
         for project in projects
     }
     expired_project_ids: Set[str] = set()
-    info = fetch_mutil_projects_info(project_ids=[project.id for project in projects])
+    db_project_ids = [project.id for project in projects]
+    alive_project_ids = []
+    
+    info = fetch_mutil_projects_info(project_ids=db_project_ids)
+    
     if info is not None:
         with ModelSubmitter() as submitter:
             for project in info:
-                submitter.add(Project(**project))
                 project_id = project["id"]
+
+                # mark as alive
+                alive_project_ids.append(project_id)
+
+                submitter.add(Project(**project))
+
                 sync_date: datetime.datetime = project_info[project_id][
                     "sync_date"
                 ].replace(tzinfo=None)
@@ -61,7 +70,10 @@ def check_modrinth_data_updated(projects: List[Project]) -> Set[str]:
                         f"Project {project_id} is updated {sync_date.isoformat(timespec='seconds')} -> {updated_date.isoformat(timespec='seconds')}!"
                     )
 
-    return expired_project_ids
+    # check if project is not alive
+    not_alive_project_ids = set(db_project_ids) - set(alive_project_ids)
+
+    return expired_project_ids, not_alive_project_ids
 
 
 # check modrinth_project_ids queue

mcim_sync/cleaner/modrinth.py

@@ -0,0 +1,29 @@
+from typing import Union, List, Set
+
+from mcim_sync.database.mongodb import sync_mongo_engine, raw_mongo_client
+from mcim_sync.utils.loger import log
+from mcim_sync.config import Config
+
+config = Config.load()
+
+def remove_project(project_id: str):
+    project_result = raw_mongo_client["modrinth_projects"].delete_one({"_id": project_id})
+    version_result = raw_mongo_client["modrinth_versions"].delete_many({"project_id": project_id})
+    hash_result = raw_mongo_client["modrinth_hashes"].delete_many({"project_id": project_id})
+    log.debug(f"Remove project {project_id}, {version_result.deleted_count} versions, {hash_result.deleted_count} hashes")
+    return project_result, version_result, hash_result
+
+def remove_projects(project_ids: List[str]):
+    result = []
+    for project_id in project_ids:
+        singal_result = remove_project(project_id)
+        if singal_result[0].deleted_count == 0 and singal_result[1].deleted_count == 0 and singal_result[2].deleted_count == 0:
+            log.debug(f"Can't remove project {project_id}, not found")
+            continue
+        result.append({
+            "project_id": project_id,
+            "version_count": singal_result[1].deleted_count,
+            "hash_count": singal_result[2].deleted_count
+        })
+        log.debug(f"Remove project {project_id}, {singal_result[1].deleted_count} versions, {singal_result[2].deleted_count} hashes")
+    return result

mcim_sync/fetcher/modrinth.py

@@ -7,7 +7,7 @@
 from mcim_sync.utils.loger import log
 from mcim_sync.config import Config
 from mcim_sync.models.database.modrinth import Project
-from mcim_sync.checker.modrinth import check_modrinth_data_updated
+from mcim_sync.checker.modrinth import check_modrinth_data_updated_and_alive
 
 config = Config.load()
 
@@ -70,8 +70,9 @@ def fetch_modrinth_data_by_sync_at():
 
 
 
-def fetch_expired_modrinth_data() -> List[str]:
+def fetch_expired_and_removed_modrinth_data() -> tuple[List[str], List[str]]:
     expired_project_ids = set()
+    removed_project_ids = set()
     skip = 0
     while True:
         projects_result: List[Project] = list(
@@ -80,9 +81,10 @@ def fetch_expired_modrinth_data() -> List[str]:
         if not projects_result:
             break
         skip += MODRINTH_LIMIT_SIZE
-        check_expired_result = check_modrinth_data_updated(projects_result)
+        check_expired_result, not_alive_result = check_modrinth_data_updated_and_alive(projects_result)
         expired_project_ids.update(check_expired_result)
-        log.debug(f"Matched {len(check_expired_result)} expired projects")
+        removed_project_ids.update(not_alive_result)
+        log.debug(f"Matched {len(check_expired_result)} expired projects, {len(not_alive_result)} removed projects")
         time.sleep(MODRINTH_DELAY)
         log.debug(f"Delay {MODRINTH_DELAY} seconds")
-    return list(expired_project_ids)
+    return list(expired_project_ids), list(removed_project_ids)

mcim_sync/tasks/modrinth.py

@@ -21,7 +21,8 @@
     check_modrinth_hashes_available,
     check_newest_search_result,
 )
-from mcim_sync.fetcher.modrinth import fetch_expired_modrinth_data
+from mcim_sync.cleaner.modrinth import remove_projects
+from mcim_sync.fetcher.modrinth import fetch_expired_and_removed_modrinth_data
 from mcim_sync.queues.modrinth import clear_modrinth_all_queues
 from mcim_sync.tasks import create_tasks_pool, modrinth_pause_event
 
@@ -34,11 +35,18 @@ def refresh_modrinth_with_modify_date() -> bool:
     log.info("Start fetching expired Modrinth data.")
 
     if config.sync_modrinth:
-        modrinth_expired_data = fetch_expired_modrinth_data()
+        modrinth_expired_data, modrinth_removed_data = fetch_expired_and_removed_modrinth_data()
 
-        log.info(f"Modrinth expired data fetched: {len(modrinth_expired_data)}")
-        log.info("Start syncing Modrinth expired data...")
+        log.info(f"Modrinth expired data fetched: {len(modrinth_expired_data)}, removed data: {len(modrinth_removed_data)}")
+
+        # 删除已经源删除的 modrinth 数据
+        if modrinth_removed_data:
+            log.info(f"Start removing modrinth data: {len(modrinth_removed_data)}")
+            remove_projects(modrinth_removed_data)
+            log.info(f"Removed {len(modrinth_removed_data)} modrinth data.")
 
+        # 刷新过期的 modrinth 数据
+        log.info("Start syncing Modrinth expired data...")
         modrinth_pause_event.set()
         modrinth_pool, modrinth_futures = create_tasks_pool(
             sync_project,  # 需要 ProjectDetail 返回值