分批删除,每批 1000 条,删完一批 commit 一次,不会锁表
默认保留 30 天,可通过环境变量 SIGNIN_LOG_RETAIN_DAYS 调整 每天凌晨 3 点执行,错开签到高峰 有完整日志记录删了多少条、截止日期是什么 用独立 engine 避免之前的 event loop 问题
This commit is contained in:
@@ -13,6 +13,7 @@ import sys
|
|||||||
import logging
|
import logging
|
||||||
|
|
||||||
from celery import Celery
|
from celery import Celery
|
||||||
|
from celery.schedules import crontab
|
||||||
|
|
||||||
# 确保 shared 模块可导入
|
# 确保 shared 模块可导入
|
||||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
|
||||||
@@ -42,6 +43,10 @@ celery_app.conf.update(
|
|||||||
"task": "task_scheduler.app.tasks.signin_tasks.check_and_run_due_tasks",
|
"task": "task_scheduler.app.tasks.signin_tasks.check_and_run_due_tasks",
|
||||||
"schedule": 60.0, # 每分钟检查一次(轻量查询,只在到期前 5 分钟才真正提交)
|
"schedule": 60.0, # 每分钟检查一次(轻量查询,只在到期前 5 分钟才真正提交)
|
||||||
},
|
},
|
||||||
|
"cleanup-old-logs": {
|
||||||
|
"task": "task_scheduler.app.tasks.signin_tasks.cleanup_old_signin_logs",
|
||||||
|
"schedule": crontab(hour=3, minute=0), # 每天凌晨 3 点
|
||||||
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -396,3 +396,59 @@ async def _do_single_signin(cookies: Dict[str, str], topic: dict) -> dict:
|
|||||||
return {"status": "failed", "message": f"code={code}, msg={msg}"}
|
return {"status": "failed", "message": f"code={code}, msg={msg}"}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"status": "failed", "message": str(e)}
|
return {"status": "failed", "message": str(e)}
|
||||||
|
|
||||||
|
|
||||||
|
# =============== 日志清理 ===============
|
||||||
|
|
||||||
|
# 保留天数,可通过环境变量覆盖
|
||||||
|
SIGNIN_LOG_RETAIN_DAYS = int(os.getenv("SIGNIN_LOG_RETAIN_DAYS", "30"))
|
||||||
|
# 每批删除条数,避免长时间锁表
|
||||||
|
CLEANUP_BATCH_SIZE = 1000
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task
|
||||||
|
def cleanup_old_signin_logs():
|
||||||
|
"""
|
||||||
|
清理超过 N 天的签到日志。
|
||||||
|
分批删除,每批最多 CLEANUP_BATCH_SIZE 条,避免锁表。
|
||||||
|
"""
|
||||||
|
logger.info(f"🧹 开始清理 {SIGNIN_LOG_RETAIN_DAYS} 天前的签到日志...")
|
||||||
|
return _run_async(_async_cleanup())
|
||||||
|
|
||||||
|
|
||||||
|
async def _async_cleanup():
|
||||||
|
from sqlalchemy import delete, func
|
||||||
|
|
||||||
|
cutoff = datetime.now() - timedelta(days=SIGNIN_LOG_RETAIN_DAYS)
|
||||||
|
total_deleted = 0
|
||||||
|
|
||||||
|
SessionFactory, eng = _make_session()
|
||||||
|
try:
|
||||||
|
async with SessionFactory() as session:
|
||||||
|
while True:
|
||||||
|
# 分批删除:先查出一批 id,再按 id 删除
|
||||||
|
stmt = (
|
||||||
|
select(SigninLog.id)
|
||||||
|
.where(SigninLog.signed_at < cutoff)
|
||||||
|
.limit(CLEANUP_BATCH_SIZE)
|
||||||
|
)
|
||||||
|
result = await session.execute(stmt)
|
||||||
|
ids = [row[0] for row in result.all()]
|
||||||
|
|
||||||
|
if not ids:
|
||||||
|
break
|
||||||
|
|
||||||
|
del_stmt = delete(SigninLog).where(SigninLog.id.in_(ids))
|
||||||
|
await session.execute(del_stmt)
|
||||||
|
await session.commit()
|
||||||
|
total_deleted += len(ids)
|
||||||
|
logger.info(f"🧹 已删除 {total_deleted} 条...")
|
||||||
|
|
||||||
|
# 如果本批不满,说明已经删完
|
||||||
|
if len(ids) < CLEANUP_BATCH_SIZE:
|
||||||
|
break
|
||||||
|
finally:
|
||||||
|
await eng.dispose()
|
||||||
|
|
||||||
|
logger.info(f"✅ 日志清理完成,共删除 {total_deleted} 条(截止 {cutoff.strftime('%Y-%m-%d')})")
|
||||||
|
return {"deleted": total_deleted, "cutoff": cutoff.isoformat()}
|
||||||
|
|||||||
Reference in New Issue
Block a user