Skip to content

Commit 33fa04f

Browse files
z0z0r4z0z0r4
committed
feat: 允许在配置文件逐个控制 Job
1 parent 5741cbe commit 33fa04f

File tree

4 files changed

+174
-146
lines changed

4 files changed

+174
-146
lines changed

‎mcim_sync/config.py‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import json
22
import os
33
from typing import Optional, Union
4-
from pydantic import BaseModel, ValidationError, validator
4+
from pydantic import BaseModel, ValidationError, field_validator
55
from enum import Enum
66

77
# config path
@@ -23,13 +23,30 @@ class RedisConfigModel(BaseModel):
2323
password: Optional[str] = None
2424
database: int = 0
2525

26+
class JobConfigModel(BaseModel):
27+
curseforge_refresh: bool = True
28+
modrinth_refresh: bool = True
29+
sync_curseforge_by_queue: bool = True
30+
sync_curseforge_by_search: bool = True
31+
sync_modrinth_by_queue: bool = True
32+
sync_modrinth_by_search: bool = True
33+
curseforge_categories: bool = True
34+
modrinth_tags: bool = True
35+
global_statistics: bool = True
36+
37+
@field_validator('*', pre=True)
38+
def validate_bool(cls, v):
39+
if isinstance(v, str):
40+
return v.lower() in ('true', '1')
41+
return bool(v)
42+
2643

2744
class JobInterval(BaseModel):
2845
curseforge_refresh: int = 60 * 60 * 2 # 2 hours
2946
modrinth_refresh: int = 60 * 60 * 2 # 2 hours
30-
sync_curseforge: int = 60 * 5 # 5 minutes
47+
sync_curseforge_by_queue: int = 60 * 5 # 5 minutes
3148
sync_curseforge_by_search: int = 60 * 60 * 2 # 2 hours
32-
sync_modrinth: int = 60 * 5 # 5 minutes
49+
sync_modrinth_by_queue: int = 60 * 5 # 5 minutes
3350
sync_modrinth_by_search: int = 60 * 60 * 2 # 2 hours
3451
curseforge_categories: int = 60 * 60 * 24 # 24 hours
3552
modrinth_tags: int = 60 * 60 * 24 # 24 hours
@@ -40,10 +57,9 @@ class ConfigModel(BaseModel):
4057
debug: bool = False
4158
mongodb: MongodbConfigModel = MongodbConfigModel()
4259
redis: RedisConfigModel = RedisConfigModel()
60+
job_config: JobConfigModel = JobConfigModel()
4361
interval: JobInterval = JobInterval()
4462
max_workers: int = 8
45-
sync_curseforge: bool = True
46-
sync_modrinth: bool = True
4763
curseforge_chunk_size: int = 1000
4864
modrinth_chunk_size: int = 1000
4965
curseforge_delay: Union[float, int] = 1

‎mcim_sync/tasks/curseforge.py‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -30,39 +30,38 @@
3030
def refresh_curseforge_with_modify_date() -> bool:
3131
log.info("Start fetching expired CurseForge data.")
3232

33-
if config.sync_curseforge:
34-
curseforge_expired_modids = fetch_expired_curseforge_data()
33+
curseforge_expired_modids = fetch_expired_curseforge_data()
3534

36-
# 到底哪来的的 wow，排除小于 30000 的 modid
37-
curseforge_expired_modids = [
38-
modid for modid in curseforge_expired_modids if modid >= 30000
39-
]
35+
# 到底哪来的的 wow，排除小于 30000 的 modid
36+
curseforge_expired_modids = [
37+
modid for modid in curseforge_expired_modids if modid >= 30000
38+
]
4039

41-
log.info(f"Curseforge expired data fetched: {len(curseforge_expired_modids)}")
42-
log.info("Start syncing CurseForge expired data...")
40+
log.info(f"Curseforge expired data fetched: {len(curseforge_expired_modids)}")
41+
log.info("Start syncing CurseForge expired data...")
4342

44-
curseforge_pause_event.set()
45-
curseforge_pool, curseforge_futures = create_tasks_pool(
46-
sync_mod, # 需要 ProjectDetail 返回值
47-
curseforge_expired_modids,
48-
MAX_WORKERS,
49-
"refresh_curseforge",
50-
)
51-
projects_detail_info = []
52-
for future in as_completed(curseforge_futures):
53-
result = future.result()
54-
if result:
55-
projects_detail_info.append(result)
56-
else:
57-
curseforge_pool.shutdown()
43+
curseforge_pause_event.set()
44+
curseforge_pool, curseforge_futures = create_tasks_pool(
45+
sync_mod, # 需要 ProjectDetail 返回值
46+
curseforge_expired_modids,
47+
MAX_WORKERS,
48+
"refresh_curseforge",
49+
)
50+
projects_detail_info = []
51+
for future in as_completed(curseforge_futures):
52+
result = future.result()
53+
if result:
54+
projects_detail_info.append(result)
55+
else:
56+
curseforge_pool.shutdown()
5857

59-
if config.telegram_bot:
60-
notification = RefreshNotification(
61-
platform=Platform.CURSEFORGE,
62-
projects_detail_info=projects_detail_info,
63-
)
64-
notification.send_to_telegram()
65-
log.info("CurseForge refresh message sent to telegram.")
58+
if config.telegram_bot:
59+
notification = RefreshNotification(
60+
platform=Platform.CURSEFORGE,
61+
projects_detail_info=projects_detail_info,
62+
)
63+
notification.send_to_telegram()
64+
log.info("CurseForge refresh message sent to telegram.")
6665

6766
return True
6867

@@ -204,9 +203,10 @@ def sync_curseforge_by_search():
204203
notice.send_to_telegram()
205204

206205
log.info("All Message sent to telegram.")
207-
206+
208207
return True
209208

209+
210210
# def sync_curseforge_full():
211211
# log.info("Start fetching all data.")
212212
# total_data = {

‎mcim_sync/tasks/modrinth.py‎

Lines changed: 42 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -35,44 +35,46 @@
3535
def refresh_modrinth_with_modify_date() -> bool:
3636
log.info("Start fetching expired Modrinth data.")
3737

38-
if config.sync_modrinth:
39-
modrinth_expired_data, modrinth_removed_data = fetch_expired_and_removed_modrinth_data()
38+
modrinth_expired_data, modrinth_removed_data = (
39+
fetch_expired_and_removed_modrinth_data()
40+
)
41+
42+
log.info(
43+
f"Modrinth expired data fetched: {len(modrinth_expired_data)}, removed data: {len(modrinth_removed_data)}"
44+
)
45+
46+
# 删除已经源删除的 modrinth 数据
47+
if modrinth_removed_data:
48+
log.info(f"Start removing modrinth data: {len(modrinth_removed_data)}")
49+
remove_projects(modrinth_removed_data)
50+
log.info(f"Removed {len(modrinth_removed_data)} modrinth data.")
51+
log.debug(f"Modrinth removed data: {modrinth_removed_data}")
52+
53+
# 刷新过期的 modrinth 数据
54+
log.info("Start syncing Modrinth expired data...")
55+
modrinth_pause_event.set()
56+
modrinth_pool, modrinth_futures = create_tasks_pool(
57+
sync_project, # 需要 ProjectDetail 返回值
58+
modrinth_expired_data,
59+
MAX_WORKERS,
60+
"refresh_modrinth",
61+
)
62+
63+
projects_detail_info = []
64+
for future in as_completed(modrinth_futures):
65+
result = future.result()
66+
if result:
67+
projects_detail_info.append(result)
68+
else:
69+
modrinth_pool.shutdown()
4070

41-
log.info(f"Modrinth expired data fetched: {len(modrinth_expired_data)}, removed data: {len(modrinth_removed_data)}")
42-
43-
# 删除已经源删除的 modrinth 数据
44-
if modrinth_removed_data:
45-
log.info(f"Start removing modrinth data: {len(modrinth_removed_data)}")
46-
remove_projects(modrinth_removed_data)
47-
log.info(f"Removed {len(modrinth_removed_data)} modrinth data.")
48-
log.debug(f"Modrinth removed data: {modrinth_removed_data}")
49-
50-
51-
# 刷新过期的 modrinth 数据
52-
log.info("Start syncing Modrinth expired data...")
53-
modrinth_pause_event.set()
54-
modrinth_pool, modrinth_futures = create_tasks_pool(
55-
sync_project, # 需要 ProjectDetail 返回值
56-
modrinth_expired_data,
57-
MAX_WORKERS,
58-
"refresh_modrinth",
71+
if config.telegram_bot:
72+
notification = RefreshNotification(
73+
platform=Platform.MODRINTH,
74+
projects_detail_info=projects_detail_info,
5975
)
60-
61-
projects_detail_info = []
62-
for future in as_completed(modrinth_futures):
63-
result = future.result()
64-
if result:
65-
projects_detail_info.append(result)
66-
else:
67-
modrinth_pool.shutdown()
68-
69-
if config.telegram_bot:
70-
notification = RefreshNotification(
71-
platform=Platform.MODRINTH,
72-
projects_detail_info=projects_detail_info,
73-
)
74-
notification.send_to_telegram()
75-
log.info("Modrinth refresh message sent to telegram.")
76+
notification.send_to_telegram()
77+
log.info("Modrinth refresh message sent to telegram.")
7678

7779
return True
7880

@@ -141,6 +143,7 @@ def sync_modrinth_queue() -> bool:
141143

142144
return True
143145

146+
144147
def sync_modrinth_by_search():
145148
"""
146149
从搜索接口拉取 modrinth 的 new project id
@@ -164,7 +167,9 @@ def sync_modrinth_by_search():
164167
projects_detail_info.append(result)
165168

166169
pool.shutdown()
167-
log.info(f"Modrinth sync new project by search finished, total: {len(new_project_ids)}")
170+
log.info(
171+
f"Modrinth sync new project by search finished, total: {len(new_project_ids)}"
172+
)
168173

169174
if config.telegram_bot:
170175
notice = SearchSyncNotification(

0 commit comments

Comments
 (0)