迁移目录

This commit is contained in:
hajimi
2026-06-12 22:27:18 +08:00
parent 7381097582
commit cd44cd6e47
92 changed files with 1839 additions and 0 deletions
View File
+183
View File
@@ -0,0 +1,183 @@
import json
from datetime import datetime
from typing import Any, Dict, Optional
import aiohttp
from loguru import logger
from src.core.config import get_config
from src.storage.database import Database
class CrontabAlertManager:
def __init__(self, db: Optional[Database] = None):
self._cfg = get_config().alert
self._db = db or Database()
@staticmethod
def build_dedupe_key(crontab_id: int, alert_type: str) -> str:
return f"crontab:{int(crontab_id)}:{alert_type}"
@staticmethod
def _pick_http_status(severe_http_statuses: list[int]) -> int:
statuses = [int(item) for item in severe_http_statuses if item is not None]
if not statuses:
return 0
return sorted(statuses)[0]
async def record_crawler_result(
self,
action: str,
crontab_id: int,
crontab_log_id: int,
task_result: Dict[str, Any],
output: str,
):
await self._db.ensure_alert_tables()
if action == "error_report":
return
policy = task_result.get("policy", "saved_required")
candidate_count = int(task_result.get("candidate_count") or 0)
saved_count = int(task_result.get("saved_count") or 0)
http_error_count = int(task_result.get("http_error_count") or 0)
severe_http_statuses = list(task_result.get("severe_http_statuses") or [])
execution_success = bool(task_result.get("execution_success", True))
success = bool(task_result.get("success"))
error_message = str(task_result.get("error_message") or "")
summary_text = str(task_result.get("summary_text") or "")
alert_type = ""
http_status = 0
if http_error_count > 0:
alert_type = "http_error"
http_status = self._pick_http_status(severe_http_statuses)
elif not execution_success or (not success and error_message):
alert_type = "task_exception"
elif policy == "saved_required" and saved_count <= 0:
alert_type = "no_data"
elif policy == "saved_if_candidates" and candidate_count > 0 and saved_count <= 0:
alert_type = "no_data"
if not alert_type:
await self._db.resolve_crontab_alerts(crontab_id)
return
detail = {
"summary_text": summary_text[:1000],
"candidate_count": candidate_count,
"saved_count": saved_count,
"http_error_count": http_error_count,
"severe_http_statuses": severe_http_statuses,
"error_message": error_message[:1000],
"output_excerpt": output[:2000],
}
await self._db.upsert_crontab_alert(
crontab_id=crontab_id,
crontab_log_id=crontab_log_id,
task_name=str(task_result.get("task_name") or action),
command=str(task_result.get("command") or "crawler"),
params=str(task_result.get("params") or action),
alert_type=alert_type,
http_status=http_status,
detail=detail,
dedupe_key=self.build_dedupe_key(crontab_id, alert_type),
)
async def dispatch_open_alerts(self, limit: int = 50) -> Dict[str, Any]:
await self._db.ensure_alert_tables()
alerts = await self._db.get_dispatchable_crontab_alerts(self._cfg.cooldown_seconds, limit=limit)
if not alerts:
return {
"success": True,
"count": 0,
"candidate_count": 0,
"saved_count": 0,
"summary_text": "无待发送告警",
}
if not self._cfg.dispatch_enabled or not self._cfg.wecom_webhook:
logger.warning(f"[alert] 企业微信告警未启用或 webhook 未配置,当前待发送 {len(alerts)}")
return {
"success": True,
"count": 0,
"candidate_count": len(alerts),
"saved_count": 0,
"summary_text": f"企业微信告警未启用或 webhook 未配置,待发送 {len(alerts)}",
}
sent = 0
failed = 0
for alert in alerts:
ok = await self._send_wecom_alert(alert)
if ok:
await self._db.mark_crontab_alert_notified(int(alert["id"]))
sent += 1
else:
failed += 1
return {
"success": failed == 0,
"count": sent,
"candidate_count": len(alerts),
"saved_count": sent,
"failed": failed,
"summary_text": f"企业微信告警发送完成:成功 {sent} 条,失败 {failed} 条,待发送 {len(alerts)}",
}
async def _send_wecom_alert(self, alert: Dict[str, Any]) -> bool:
message = self._build_markdown(alert)
payload = {
"msgtype": "markdown",
"markdown": {"content": message},
}
timeout = aiohttp.ClientTimeout(total=max(3, int(self._cfg.request_timeout)))
try:
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.post(self._cfg.wecom_webhook, json=payload) as resp:
body = await resp.text()
if resp.status != 200:
logger.warning(f"[alert] 企业微信发送失败 HTTP {resp.status}: {body[:300]}")
return False
data = json.loads(body)
if int(data.get("errcode", -1)) != 0:
logger.warning(f"[alert] 企业微信返回失败: {body[:300]}")
return False
return True
except Exception as e:
logger.warning(f"[alert] 企业微信发送异常: {e}")
return False
@staticmethod
def _build_markdown(alert: Dict[str, Any]) -> str:
detail_raw = alert.get("detail") or ""
try:
detail = json.loads(detail_raw) if isinstance(detail_raw, str) else (detail_raw or {})
except Exception:
detail = {"summary_text": str(detail_raw)}
def fmt_ts(value: Any) -> str:
try:
ts = int(value or 0)
if ts <= 0:
return "-"
return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
except Exception:
return "-"
severe = detail.get("severe_http_statuses") or []
http_text = "/".join(str(item) for item in severe) if severe else (str(alert.get("http_status") or "-"))
summary = str(detail.get("summary_text") or detail.get("error_message") or "无摘要").replace("\n", " ")
return "\n".join([
"### Sport Era 定时任务告警",
f"> 任务:`{alert.get('task_name', '')}` / #{alert.get('crontab_id', 0)}",
f"> 命令:`{alert.get('command', '')} {alert.get('params', '')}`",
f"> 规则:`{alert.get('expression') or '-'}`",
f"> 类型:`{alert.get('alert_type', '')}`",
f"> HTTP`{http_text}`",
f"> 保存:`{detail.get('saved_count', 0)}` / 候选:`{detail.get('candidate_count', 0)}` / 请求错误:`{detail.get('http_error_count', 0)}`",
f"> 首次发生:`{fmt_ts(alert.get('first_seen_at'))}`",
f"> 最近发生:`{fmt_ts(alert.get('last_seen_at'))}`",
f"> 摘要:{summary[:1000]}",
])
+266
View File
@@ -0,0 +1,266 @@
"""
配置加载模块
"""
import os
from pathlib import Path
from typing import Dict, List, Optional, Any
import yaml
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
host: str = "localhost"
port: int = 3306
database: str = ""
username: str = ""
password: str = ""
charset: str = "utf8mb4"
prefix: str = "la_"
pool_size: int = 10
class RedisConfig(BaseModel):
host: str = "localhost"
port: int = 6379
password: str = ""
db: int = 0
class DelayConfig(BaseModel):
base: float = 3.0
min: float = 2.0
max: float = 8.0
jitter: bool = True
class SessionConfig(BaseModel):
rotate_interval: int = 300
max_requests_per_session: int = 50
class CrawlerConfig(BaseModel):
mode: str = "hybrid"
max_concurrent: int = 3
request_timeout: int = 30
max_retries: int = 3
retry_backoff: float = 2.0
delay: DelayConfig = Field(default_factory=DelayConfig)
session: SessionConfig = Field(default_factory=SessionConfig)
class FingerprintConfig(BaseModel):
pool_size: int = 50
min_success_rate: float = 0.6
rotation_interval: int = 300
cooldown: int = 60
class TLSConfig(BaseModel):
enabled: bool = True
impersonate: str = "chrome124"
class ViewportConfig(BaseModel):
width: int = 1920
height: int = 1080
class AntiFeaturesConfig(BaseModel):
remove_webdriver: bool = True
fake_plugins: bool = True
fake_languages: bool = True
fake_chrome_runtime: bool = True
disable_webrtc: bool = True
random_viewport: bool = True
human_mouse: bool = True
random_scroll: bool = True
class PlaywrightAntiConfig(BaseModel):
browser: str = "chromium"
headless: bool = True
stealth: bool = True
viewport: ViewportConfig = Field(default_factory=ViewportConfig)
anti_features: AntiFeaturesConfig = Field(default_factory=AntiFeaturesConfig)
class ProxyConfig(BaseModel):
enabled: bool = False
rotate: bool = True
providers: List[str] = Field(default_factory=list)
test_url: str = "https://httpbin.org/ip"
max_failures: int = 3
class AntiDetectConfig(BaseModel):
fingerprint: FingerprintConfig = Field(default_factory=FingerprintConfig)
tls: TLSConfig = Field(default_factory=TLSConfig)
playwright: PlaywrightAntiConfig = Field(default_factory=PlaywrightAntiConfig)
proxy: ProxyConfig = Field(default_factory=ProxyConfig)
class APIEndpoints(BaseModel):
standings: str = "/soccer/biz/data/standing"
schedule: str = "/soccer/biz/data/schedule"
match_detail: str = "/soccer/biz/data/match_detail"
match_menu: str = "/api/v2/config/match_menu"
team_info: str = "/soccer/biz/data/team_info"
player_stats: str = "/soccer/biz/data/player_stats"
live_matches: str = "/soccer/biz/data/live"
match_situation: str = "/mobile/match/situation"
match_lineup: str = "/mobile/match/lineup"
news: str = "/api/v2/article/list"
class DefaultParams(BaseModel):
app: str = "dqd"
platform: str = "www"
version: str = "0"
language: str = "zh-cn"
class TargetLeague(BaseModel):
league: str
league_code: str = ""
season_id: int
priority: int = 100
active: bool = True
class DongqiudiConfig(BaseModel):
base_url: str = "https://sport-data.dongqiudi.com"
web_url: str = "https://www.dongqiudi.com"
api: APIEndpoints = Field(default_factory=APIEndpoints)
default_params: DefaultParams = Field(default_factory=DefaultParams)
targets: List[TargetLeague] = Field(default_factory=list)
class LotterySource(BaseModel):
name: str
code: str
category_id: int
url: str
class LotteryConfig(BaseModel):
sources: List[LotterySource] = Field(default_factory=list)
class SchedulerConfig(BaseModel):
standings_cron: str = "0 2 * * *"
schedule_cron: str = "0 */6 * * *"
live_cron: str = "*/5 * * * *"
news_cron: str = "*/30 * * * *"
content_cron: str = "*/10 * * * *"
video_cron: str = "*/30 * * * *"
csl_match_cron: str = "0 */1 * * *"
nba_match_cron: str = "0 */1 * * *"
cba_match_cron: str = "0 */1 * * *"
epl_match_cron: str = "0 */1 * * *"
bundesliga_match_cron: str = "0 */1 * * *"
laliga_match_cron: str = "0 */1 * * *"
seriea_match_cron: str = "0 */1 * * *"
ligue1_match_cron: str = "0 */1 * * *"
ucl_match_cron: str = "0 */1 * * *"
uel_match_cron: str = "0 */1 * * *"
tennis_match_cron: str = "0 */1 * * *"
esports_match_cron: str = "0 */1 * * *"
sports_match_cron: str = "0 */1 * * *"
league_news_cron: str = "*/30 * * * *"
article_content_cron: str = "*/1 * * * *"
class LoggingConfig(BaseModel):
level: str = "INFO"
file: str = "logs/crawler.log"
max_size_mb: int = 50
backup_count: int = 5
format: str = "{time:YYYY-MM-DD HH:mm:ss} | {level:<8} | {name}:{function}:{line} - {message}"
class AlertConfig(BaseModel):
dispatch_enabled: bool = False
wecom_webhook: str = ""
cooldown_seconds: int = 1800
request_timeout: int = 10
class AppConfig(BaseModel):
database: DatabaseConfig = Field(default_factory=DatabaseConfig)
redis: RedisConfig = Field(default_factory=RedisConfig)
crawler: CrawlerConfig = Field(default_factory=CrawlerConfig)
anti_detect: AntiDetectConfig = Field(default_factory=AntiDetectConfig)
dongqiudi: DongqiudiConfig = Field(default_factory=DongqiudiConfig)
lottery: LotteryConfig = Field(default_factory=LotteryConfig)
scheduler: SchedulerConfig = Field(default_factory=SchedulerConfig)
logging: LoggingConfig = Field(default_factory=LoggingConfig)
alert: AlertConfig = Field(default_factory=AlertConfig)
_config: Optional[AppConfig] = None
def load_config(config_path: Optional[str] = None) -> AppConfig:
global _config
if config_path is None:
config_path = os.environ.get("DQD_CONFIG_PATH")
if config_path is None:
candidates = [
Path(__file__).parent.parent.parent / "config" / "settings.yaml",
Path("config/settings.yaml"),
]
for p in candidates:
if p.exists():
config_path = str(p)
break
if config_path and Path(config_path).exists():
with open(config_path, "r", encoding="utf-8") as f:
raw = yaml.safe_load(f) or {}
raw = _apply_env_overrides(raw)
_config = AppConfig(**raw)
else:
_config = AppConfig(**_apply_env_overrides({}))
return _config
def _apply_env_overrides(raw: Dict[str, Any]) -> Dict[str, Any]:
mapping = {
"DQD_DB_HOST": ("database", "host", str),
"DQD_DB_PORT": ("database", "port", int),
"DQD_DB_NAME": ("database", "database", str),
"DQD_DB_USER": ("database", "username", str),
"DQD_DB_PASSWORD": ("database", "password", str),
"DQD_DB_PREFIX": ("database", "prefix", str),
"DQD_DB_CHARSET": ("database", "charset", str),
"DQD_DB_POOL_SIZE": ("database", "pool_size", int),
"DQD_REDIS_HOST": ("redis", "host", str),
"DQD_REDIS_PORT": ("redis", "port", int),
"DQD_REDIS_PASSWORD": ("redis", "password", str),
"DQD_REDIS_DB": ("redis", "db", int),
"DQD_LOG_FILE": ("logging", "file", str),
"DQD_LOG_LEVEL": ("logging", "level", str),
}
for env_name, (section, key, caster) in mapping.items():
if env_name not in os.environ:
continue
value = os.environ.get(env_name, "")
try:
cast_value = caster(value)
except (TypeError, ValueError):
continue
raw.setdefault(section, {})
raw[section][key] = cast_value
return raw
def get_config() -> AppConfig:
global _config
if _config is None:
_config = load_config()
return _config
@@ -0,0 +1,93 @@
"""
全局错误收集器
- 收集所有HTTP请求错误,异步写入数据库
- 不阻塞主流程,写入失败只打日志
"""
import asyncio
from typing import Optional
from loguru import logger
class ErrorCollector:
_instance: Optional["ErrorCollector"] = None
_task_name: str = ""
def __init__(self):
self._db = None
self.reset_run_stats()
@classmethod
def get(cls) -> "ErrorCollector":
if cls._instance is None:
cls._instance = cls()
return cls._instance
def set_task_name(self, name: str):
self._task_name = name
def reset_run_stats(self):
self._http_error_count = 0
self._severe_http_statuses = set()
self._recent_errors = []
def get_run_stats(self) -> dict:
return {
"http_error_count": self._http_error_count,
"severe_http_statuses": sorted(self._severe_http_statuses),
"recent_errors": list(self._recent_errors),
}
def _record_run_error(self, request_url: str, http_status: int, error_type: str, error_message: str):
should_count = http_status == 404 or http_status == 0 or http_status >= 500
if should_count:
self._http_error_count += 1
self._severe_http_statuses.add(int(http_status))
if len(self._recent_errors) < 10:
self._recent_errors.append({
"request_url": request_url[:300],
"http_status": int(http_status),
"error_type": error_type[:100],
"error_message": error_message[:500],
})
async def _get_db(self):
if self._db is None:
from src.storage.database import Database
self._db = Database()
await self._db.connect()
return self._db
async def report(
self,
request_url: str,
request_method: str = "GET",
request_params: str = "",
http_status: int = 0,
error_type: str = "",
error_message: str = "",
response_body: str = "",
channel: str = "",
):
try:
self._record_run_error(request_url, http_status, error_type, error_message)
db = await self._get_db()
await db.insert_error_log(
task_name=self._task_name,
request_url=request_url,
request_method=request_method,
request_params=request_params,
http_status=http_status,
error_type=error_type,
error_message=error_message,
response_body=response_body,
channel=channel,
)
except Exception as e:
logger.debug(f"[error_collector] 写入错误日志失败: {e}")
async def close(self):
if self._db:
await self._db.close()
self._db = None
+45
View File
@@ -0,0 +1,45 @@
"""
异常定义
"""
class CrawlerError(Exception):
pass
class RateLimitError(CrawlerError):
def __init__(self, message="频率限制", retry_after: int = 60):
self.retry_after = retry_after
super().__init__(message)
class BlockedError(CrawlerError):
pass
class CaptchaError(CrawlerError):
pass
class NetworkError(CrawlerError):
pass
class ParsingError(CrawlerError):
pass
class FingerprintError(CrawlerError):
pass
class ProxyError(CrawlerError):
pass
class AntiDetectError(CrawlerError):
pass
class SessionExpiredError(CrawlerError):
pass
+36
View File
@@ -0,0 +1,36 @@
"""
日志模块 - 基于 loguru
"""
import sys
from pathlib import Path
from loguru import logger
from src.core.config import get_config
def setup_logger():
cfg = get_config().logging
logger.remove()
logger.add(
sys.stderr,
level=cfg.level,
format=cfg.format,
colorize=True,
)
log_path = Path(cfg.file)
log_path.parent.mkdir(parents=True, exist_ok=True)
logger.add(
str(log_path),
level=cfg.level,
format=cfg.format,
rotation=f"{cfg.max_size_mb} MB",
retention=cfg.backup_count,
encoding="utf-8",
enqueue=True,
)
return logger