迁移目录

docker/crawler/src/anti_detect/fingerprint.py

@@ -0,0 +1,237 @@
+"""
+浏览器指纹生成与管理
+- 生成逼真的浏览器指纹（UA、屏幕、Canvas、WebGL、TLS）
+- 指纹池轮换、冷却、黑名单机制
+"""
+import asyncio
+import hashlib
+import random
+import time
+from datetime import datetime, timedelta
+from typing import Dict, Any, List, Optional, Set
+
+from loguru import logger
+from fake_useragent import UserAgent
+
+from src.core.config import get_config
+
+
+# ── 常量 ──────────────────────────────────────────────
+
+CHROME_VERSIONS = [
+    "120.0.6099.109", "121.0.6167.85", "122.0.6261.57",
+    "123.0.6312.86", "124.0.6367.91", "125.0.6422.60",
+]
+
+SCREEN_RESOLUTIONS = [
+    {"width": 1920, "height": 1080, "ratio": 1.0},
+    {"width": 1366, "height": 768, "ratio": 1.0},
+    {"width": 1536, "height": 864, "ratio": 1.25},
+    {"width": 2560, "height": 1440, "ratio": 1.5},
+    {"width": 1440, "height": 900, "ratio": 1.0},
+    {"width": 1680, "height": 1050, "ratio": 1.0},
+    {"width": 1280, "height": 800, "ratio": 1.0},
+    {"width": 3840, "height": 2160, "ratio": 2.0},
+]
+
+WEBGL_RENDERERS = [
+    "ANGLE (Intel, Intel(R) UHD Graphics 630 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (Intel, Intel(R) UHD Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (AMD, AMD Radeon(TM) Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (NVIDIA, NVIDIA GeForce GTX 1060 6GB Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (Intel, Intel(R) Iris(R) Xe Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
+]
+
+FONTS_POOL = [
+    "Arial", "Arial Black", "Calibri", "Cambria", "Consolas",
+    "Courier New", "Georgia", "Impact", "Segoe UI", "Tahoma",
+    "Times New Roman", "Trebuchet MS", "Verdana",
+    "Microsoft YaHei", "SimHei", "SimSun", "NSimSun", "FangSong", "KaiTi",
+]
+
+SEC_CH_UA_TEMPLATES = {
+    "chrome": '"Chromium";v="{major}", "Google Chrome";v="{major}", "Not-A.Brand";v="99"',
+    "edge": '"Chromium";v="{major}", "Microsoft Edge";v="{major}", "Not-A.Brand";v="99"',
+}
+
+
+class FingerprintGenerator:
+    """指纹生成器"""
+
+    def __init__(self):
+        try:
+            self._ua = UserAgent(browsers=["chrome", "edge"], os=["windows", "macos"])
+        except Exception:
+            self._ua = None
+
+    def generate(self) -> Dict[str, Any]:
+        browser = random.choice(["chrome", "edge"])
+        version = random.choice(CHROME_VERSIONS)
+        major = version.split(".")[0]
+        ua_string = self._build_ua(browser, version)
+        screen = random.choice(SCREEN_RESOLUTIONS)
+        fp_id = f"fp_{int(time.time())}_{random.randint(1000, 9999)}"
+
+        return {
+            "id": fp_id,
+            "browser": browser,
+            "version": version,
+            "user_agent": ua_string,
+            "screen": screen,
+            "sec_ch_ua": SEC_CH_UA_TEMPLATES.get(browser, SEC_CH_UA_TEMPLATES["chrome"]).format(major=major),
+            "http_headers": self._build_headers(ua_string, browser, major),
+            "canvas_hash": hashlib.md5(f"canvas_{random.randint(100000, 999999)}".encode()).hexdigest()[:16],
+            "webgl": {
+                "vendor": "Google Inc. (Intel)",
+                "renderer": random.choice(WEBGL_RENDERERS),
+            },
+            "fonts": sorted(random.sample(FONTS_POOL, k=random.randint(10, len(FONTS_POOL)))),
+            "timezone": "Asia/Shanghai",
+            "language": "zh-CN",
+            "created_at": datetime.now().isoformat(),
+            "success_rate": 1.0,
+            "usage_count": 0,
+            "is_active": True,
+        }
+
+    def _build_ua(self, browser: str, version: str) -> str:
+        os_strings = [
+            "Windows NT 10.0; Win64; x64",
+            "Windows NT 10.0; Win64; x64",
+            "Macintosh; Intel Mac OS X 10_15_7",
+        ]
+        os_str = random.choice(os_strings)
+        major = version.split(".")[0]
+
+        if browser == "edge":
+            return (
+                f"Mozilla/5.0 ({os_str}) AppleWebKit/537.36 "
+                f"(KHTML, like Gecko) Chrome/{version} Safari/537.36 Edg/{version}"
+            )
+        return (
+            f"Mozilla/5.0 ({os_str}) AppleWebKit/537.36 "
+            f"(KHTML, like Gecko) Chrome/{version} Safari/537.36"
+        )
+
+    def _build_headers(self, ua: str, browser: str, major: str) -> Dict[str, str]:
+        headers = {
+            "User-Agent": ua,
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Referer": "https://www.dongqiudi.com/",
+            "Origin": "https://www.dongqiudi.com",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-site",
+            "Sec-Ch-Ua": SEC_CH_UA_TEMPLATES.get(browser, SEC_CH_UA_TEMPLATES["chrome"]).format(major=major),
+            "Sec-Ch-Ua-Mobile": "?0",
+            "Sec-Ch-Ua-Platform": '"Windows"',
+        }
+        return headers
+
+
+class FingerprintPool:
+    """指纹池管理器"""
+
+    def __init__(self):
+        cfg = get_config().anti_detect.fingerprint
+        self._pool_size = cfg.pool_size
+        self._min_success_rate = cfg.min_success_rate
+        self._cooldown = cfg.cooldown
+        self._generator = FingerprintGenerator()
+        self._pool: Dict[str, Dict[str, Any]] = {}
+        self._blacklist: Set[str] = set()
+        self._lock = asyncio.Lock()
+
+    async def initialize(self, size: Optional[int] = None):
+        size = size or self._pool_size
+        async with self._lock:
+            for _ in range(size):
+                fp = self._generator.generate()
+                self._pool[fp["id"]] = fp
+        logger.info(f"指纹池初始化完成，共 {len(self._pool)} 个指纹")
+
+    async def acquire(self) -> Dict[str, Any]:
+        async with self._lock:
+            now = datetime.now()
+            candidates = []
+
+            for fp_id, fp in self._pool.items():
+                if fp_id in self._blacklist or not fp.get("is_active", True):
+                    continue
+                if fp.get("success_rate", 1.0) < self._min_success_rate:
+                    fp["is_active"] = False
+                    continue
+                last_used = fp.get("last_used")
+                if last_used:
+                    last_dt = datetime.fromisoformat(last_used) if isinstance(last_used, str) else last_used
+                    if (now - last_dt).total_seconds() < self._cooldown:
+                        continue
+                candidates.append(fp)
+
+            if not candidates:
+                logger.warning("指纹池耗尽，生成新指纹")
+                fp = self._generator.generate()
+                self._pool[fp["id"]] = fp
+                candidates = [fp]
+
+            candidates.sort(key=lambda x: (x.get("usage_count", 0), -x.get("success_rate", 1.0)))
+            chosen = candidates[0]
+            chosen["last_used"] = now.isoformat()
+            chosen["usage_count"] = chosen.get("usage_count", 0) + 1
+            return chosen.copy()
+
+    async def mark_success(self, fp_id: str):
+        async with self._lock:
+            fp = self._pool.get(fp_id)
+            if not fp:
+                return
+            total = fp.get("usage_count", 1)
+            fails = fp.get("fail_count", 0)
+            fp["success_rate"] = 1.0 - (fails / total) if total > 0 else 1.0
+
+    async def mark_failure(self, fp_id: str, error_type: str = ""):
+        async with self._lock:
+            fp = self._pool.get(fp_id)
+            if not fp:
+                return
+            fp["fail_count"] = fp.get("fail_count", 0) + 1
+            total = fp.get("usage_count", 1)
+            fails = fp["fail_count"]
+            fp["success_rate"] = 1.0 - (fails / total) if total > 0 else 0.0
+
+            if error_type in ("ip_blocked", "captcha", "fingerprint_detected"):
+                self._blacklist.add(fp_id)
+                fp["is_active"] = False
+                logger.warning(f"指纹 {fp_id} 被加入黑名单: {error_type}")
+
+    async def cleanup(self):
+        async with self._lock:
+            to_remove = [
+                fp_id for fp_id, fp in self._pool.items()
+                if not fp.get("is_active") and fp.get("success_rate", 1.0) < self._min_success_rate * 0.5
+            ]
+            for fp_id in to_remove:
+                self._pool.pop(fp_id, None)
+                self._blacklist.discard(fp_id)
+
+            deficit = self._pool_size - len(self._pool)
+            if deficit > 0:
+                for _ in range(deficit):
+                    fp = self._generator.generate()
+                    self._pool[fp["id"]] = fp
+                logger.info(f"指纹池补充 {deficit} 个新指纹")
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        active = sum(1 for fp in self._pool.values() if fp.get("is_active", True))
+        rates = [fp.get("success_rate", 1.0) for fp in self._pool.values()]
+        return {
+            "total": len(self._pool),
+            "active": active,
+            "blacklisted": len(self._blacklist),
+            "avg_success_rate": sum(rates) / len(rates) if rates else 1.0,
+        }

docker/crawler/src/anti_detect/proxy_manager.py

@@ -0,0 +1,162 @@
+"""
+代理管理器
+- 代理池维护、健康检测、自动轮换
+- 支持 HTTP / SOCKS5 代理
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, List, Optional
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import ProxyError
+
+
+class Proxy:
+    __slots__ = (
+        "url", "protocol", "host", "port", "username", "password",
+        "success_count", "fail_count", "last_used", "last_check",
+        "latency", "is_active",
+    )
+
+    def __init__(self, url: str, protocol: str = "http"):
+        self.url = url
+        self.protocol = protocol
+        parts = url.replace("http://", "").replace("https://", "").replace("socks5://", "")
+        auth_host = parts.split("@")
+        if len(auth_host) == 2:
+            auth = auth_host[0].split(":")
+            self.username = auth[0]
+            self.password = auth[1] if len(auth) > 1 else ""
+            host_port = auth_host[1].split(":")
+        else:
+            self.username = ""
+            self.password = ""
+            host_port = auth_host[0].split(":")
+        self.host = host_port[0]
+        self.port = int(host_port[1]) if len(host_port) > 1 else 80
+
+        self.success_count = 0
+        self.fail_count = 0
+        self.last_used = 0.0
+        self.last_check = 0.0
+        self.latency = 0.0
+        self.is_active = True
+
+    @property
+    def success_rate(self) -> float:
+        total = self.success_count + self.fail_count
+        return self.success_count / total if total > 0 else 1.0
+
+    def mark_success(self, latency: float):
+        self.success_count += 1
+        self.latency = latency
+        self.last_used = time.time()
+
+    def mark_failure(self):
+        self.fail_count += 1
+        self.last_used = time.time()
+        cfg = get_config().anti_detect.proxy
+        if self.fail_count >= cfg.max_failures:
+            self.is_active = False
+            logger.warning(f"代理 {self.host}:{self.port} 因连续失败被停用")
+
+
+class ProxyManager:
+    """代理池管理器"""
+
+    def __init__(self):
+        self._proxies: List[Proxy] = []
+        self._lock = asyncio.Lock()
+        self._enabled = get_config().anti_detect.proxy.enabled
+
+    @property
+    def enabled(self) -> bool:
+        return self._enabled and len(self._proxies) > 0
+
+    async def add_proxy(self, url: str, protocol: str = "http"):
+        async with self._lock:
+            proxy = Proxy(url, protocol)
+            self._proxies.append(proxy)
+            logger.info(f"添加代理: {proxy.host}:{proxy.port}")
+
+    async def add_proxies(self, urls: List[str], protocol: str = "http"):
+        for url in urls:
+            await self.add_proxy(url, protocol)
+
+    async def get_proxy(self) -> Optional[str]:
+        if not self._enabled:
+            return None
+
+        async with self._lock:
+            active = [p for p in self._proxies if p.is_active]
+            if not active:
+                logger.warning("无可用代理")
+                return None
+
+            active.sort(key=lambda p: (p.success_rate, -p.latency), reverse=True)
+
+            top_n = max(1, len(active) // 3)
+            chosen = random.choice(active[:top_n])
+            return chosen.url
+
+    async def mark_success(self, proxy_url: str, latency: float):
+        async with self._lock:
+            for p in self._proxies:
+                if p.url == proxy_url:
+                    p.mark_success(latency)
+                    break
+
+    async def mark_failure(self, proxy_url: str):
+        async with self._lock:
+            for p in self._proxies:
+                if p.url == proxy_url:
+                    p.mark_failure()
+                    break
+
+    async def health_check(self):
+        """批量检测代理健康状态"""
+        import aiohttp
+
+        test_url = get_config().anti_detect.proxy.test_url
+        tasks = []
+
+        async def _check(proxy: Proxy):
+            try:
+                start = time.time()
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(
+                        test_url,
+                        proxy=proxy.url,
+                        timeout=aiohttp.ClientTimeout(total=10),
+                    ) as resp:
+                        if resp.status == 200:
+                            proxy.latency = time.time() - start
+                            proxy.is_active = True
+                            proxy.last_check = time.time()
+                        else:
+                            proxy.is_active = False
+            except Exception:
+                proxy.is_active = False
+
+        async with self._lock:
+            for p in self._proxies:
+                tasks.append(_check(p))
+
+        await asyncio.gather(*tasks, return_exceptions=True)
+        active_count = sum(1 for p in self._proxies if p.is_active)
+        logger.info(f"代理健康检查完成: {active_count}/{len(self._proxies)} 可用")
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        active = sum(1 for p in self._proxies if p.is_active)
+        return {
+            "total": len(self._proxies),
+            "active": active,
+            "avg_latency": (
+                sum(p.latency for p in self._proxies if p.is_active) / active
+                if active else 0
+            ),
+        }

docker/crawler/src/anti_detect/stealth_browser.py

@@ -0,0 +1,285 @@
+"""
+Playwright 隐身浏览器
+- 深度反检测：移除 webdriver 痕迹、模拟人类行为
+- 自动处理验证码页面、JS 挑战
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, Optional, List
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import CaptchaError, BlockedError
+
+
+class StealthBrowser:
+    """隐身浏览器 - Playwright + 深度反检测"""
+
+    def __init__(self):
+        cfg = get_config().anti_detect.playwright
+        self._browser_type = cfg.browser
+        self._headless = cfg.headless
+        self._use_stealth = cfg.stealth
+        self._viewport = cfg.viewport
+        self._anti = cfg.anti_features
+        self._playwright = None
+        self._browser = None
+        self._context = None
+        self._running = False
+
+    async def start(self):
+        if self._running:
+            return
+
+        from playwright.async_api import async_playwright
+
+        self._playwright = await async_playwright().start()
+
+        launch_args = [
+            "--disable-blink-features=AutomationControlled",
+            "--disable-dev-shm-usage",
+            "--no-sandbox",
+            "--disable-infobars",
+            "--disable-background-timer-throttling",
+            "--disable-backgrounding-occluded-windows",
+            "--disable-renderer-backgrounding",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+
+        if self._anti.disable_webrtc:
+            launch_args.append("--disable-webrtc")
+
+        browser_launcher = getattr(self._playwright, self._browser_type)
+        self._browser = await browser_launcher.launch(
+            headless=self._headless,
+            args=launch_args,
+        )
+
+        context_opts = {
+            "viewport": {"width": self._viewport.width, "height": self._viewport.height},
+            "locale": "zh-CN",
+            "timezone_id": "Asia/Shanghai",
+        }
+
+        if self._anti.random_viewport:
+            w_offset = random.randint(-50, 50)
+            h_offset = random.randint(-30, 30)
+            context_opts["viewport"] = {
+                "width": self._viewport.width + w_offset,
+                "height": self._viewport.height + h_offset,
+            }
+
+        self._context = await self._browser.new_context(**context_opts)
+        self._running = True
+        logger.info(f"隐身浏览器已启动 ({self._browser_type}, headless={self._headless})")
+
+    async def stop(self):
+        if self._context:
+            await self._context.close()
+            self._context = None
+        if self._browser:
+            await self._browser.close()
+            self._browser = None
+        if self._playwright:
+            await self._playwright.stop()
+            self._playwright = None
+        self._running = False
+        logger.info("隐身浏览器已关闭")
+
+    async def new_page(self):
+        if not self._running:
+            await self.start()
+
+        page = await self._context.new_page()
+
+        if self._use_stealth:
+            await self._apply_stealth(page)
+
+        return page
+
+    async def _apply_stealth(self, page):
+        """应用全套反检测措施"""
+        try:
+            from playwright_stealth import stealth_async
+            await stealth_async(page)
+            logger.debug("playwright-stealth 反检测已应用")
+        except ImportError:
+            logger.debug("playwright-stealth 未安装，使用内置反检测")
+
+        await self._inject_anti_detection(page)
+
+    async def _inject_anti_detection(self, page):
+        """注入反检测 JS"""
+
+        if self._anti.remove_webdriver:
+            await page.add_init_script("""
+                Object.defineProperty(navigator, 'webdriver', {
+                    get: () => undefined,
+                });
+                // 删除 __webdriver_evaluate 等属性
+                delete navigator.__webdriver_evaluate;
+                delete navigator.__webdriver_unwrap;
+            """)
+
+        if self._anti.fake_plugins:
+            await page.add_init_script("""
+                Object.defineProperty(navigator, 'plugins', {
+                    get: () => {
+                        const plugins = [
+                            {name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer'},
+                            {name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai'},
+                            {name: 'Native Client', filename: 'internal-nacl-plugin'},
+                        ];
+                        plugins.length = 3;
+                        return plugins;
+                    },
+                });
+                Object.defineProperty(navigator, 'mimeTypes', {
+                    get: () => {
+                        const mimes = [
+                            {type: 'application/pdf', suffixes: 'pdf'},
+                            {type: 'application/x-nacl', suffixes: ''},
+                        ];
+                        mimes.length = 2;
+                        return mimes;
+                    },
+                });
+            """)
+
+        if self._anti.fake_languages:
+            await page.add_init_script("""
+                Object.defineProperty(navigator, 'languages', {
+                    get: () => ['zh-CN', 'zh', 'en-US', 'en'],
+                });
+                Object.defineProperty(navigator, 'language', {
+                    get: () => 'zh-CN',
+                });
+            """)
+
+        if self._anti.fake_chrome_runtime:
+            await page.add_init_script("""
+                window.chrome = {
+                    app: { isInstalled: false, InstallState: {DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed'}, RunningState: {CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running'} },
+                    runtime: { OnInstalledReason: {CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update'}, OnRestartRequiredReason: {APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic'}, PlatformArch: {ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64'}, PlatformNaclArch: {ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64'}, PlatformOs: {ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win'}, RequestUpdateCheckStatus: {NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available'} },
+                };
+            """)
+
+        # 隐藏自动化 permissions
+        await page.add_init_script("""
+            const originalQuery = window.navigator.permissions.query;
+            window.navigator.permissions.query = (parameters) => (
+                parameters.name === 'notifications' ?
+                    Promise.resolve({ state: Notification.permission }) :
+                    originalQuery(parameters)
+            );
+        """)
+
+        # 伪造 connection.rtt
+        await page.add_init_script("""
+            if (navigator.connection) {
+                Object.defineProperty(navigator.connection, 'rtt', { get: () => 50 });
+            }
+        """)
+
+    async def human_scroll(self, page, times: int = 3):
+        """模拟人类滚动行为"""
+        for _ in range(times):
+            scroll_y = random.randint(100, 400)
+            await page.mouse.wheel(0, scroll_y)
+            await asyncio.sleep(random.uniform(0.3, 1.0))
+
+    async def human_mouse_move(self, page, x: int, y: int):
+        """模拟人类鼠标移动（贝塞尔曲线）"""
+        steps = random.randint(10, 25)
+        current_x, current_y = 0, 0
+        for i in range(steps):
+            t = (i + 1) / steps
+            next_x = int(current_x + (x - current_x) * t + random.randint(-5, 5))
+            next_y = int(current_y + (y - current_y) * t + random.randint(-3, 3))
+            await page.mouse.move(next_x, next_y)
+            await asyncio.sleep(random.uniform(0.01, 0.05))
+
+    async def fetch_page(
+        self,
+        url: str,
+        wait_selector: Optional[str] = None,
+        wait_time: float = 0,
+        extract_json: bool = False,
+    ) -> Dict[str, Any]:
+        """使用隐身浏览器获取页面"""
+        page = None
+        try:
+            page = await self.new_page()
+            start = time.time()
+
+            await page.goto(url, wait_until="networkidle", timeout=30000)
+            nav_time = time.time() - start
+
+            if wait_selector:
+                await page.wait_for_selector(wait_selector, timeout=10000)
+
+            if self._anti.human_mouse:
+                await self.human_mouse_move(page, random.randint(200, 800), random.randint(200, 600))
+
+            if self._anti.random_scroll:
+                await self.human_scroll(page, times=random.randint(1, 3))
+
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+
+            content = await page.content()
+            title = await page.title()
+            total_time = time.time() - start
+
+            # 检测反爬页面
+            if self._is_blocked_page(content, title):
+                raise BlockedError(f"检测到反爬页面: {title}")
+
+            result = {
+                "success": True,
+                "url": page.url,
+                "title": title,
+                "content": content,
+                "content_size": len(content),
+                "nav_time": nav_time,
+                "total_time": total_time,
+            }
+
+            if extract_json:
+                json_data = await self._extract_json(page)
+                if json_data:
+                    result["json_data"] = json_data
+
+            return result
+
+        except (BlockedError, CaptchaError):
+            raise
+        except Exception as e:
+            logger.error(f"页面获取失败 {url}: {e}")
+            return {"success": False, "url": url, "error": str(e)}
+        finally:
+            if page:
+                await page.close()
+
+    def _is_blocked_page(self, content: str, title: str) -> bool:
+        blocked_keywords = ["验证码", "Captcha", "Access Denied", "403 Forbidden", "请稍后", "访问频繁"]
+        text = (content[:3000] + title).lower()
+        return any(kw.lower() in text for kw in blocked_keywords)
+
+    async def _extract_json(self, page) -> Optional[Any]:
+        """从页面中提取 JSON 数据（用于 API 页面）"""
+        try:
+            text = await page.evaluate("() => document.body.innerText")
+            import json
+            return json.loads(text)
+        except Exception:
+            return None
+
+    async def __aenter__(self):
+        await self.start()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.stop()

docker/crawler/src/anti_detect/tls_client.py

@@ -0,0 +1,148 @@
+"""
+TLS 指纹伪装客户端
+使用 curl_cffi 模拟真实浏览器的 TLS 握手指纹 (JA3/JA4)
+这是绕过懂球帝 TLS 指纹检测的核心模块
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, Optional
+from urllib.parse import urlencode
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import NetworkError, RateLimitError, BlockedError, CaptchaError
+
+
+# curl_cffi 支持的浏览器指纹列表
+IMPERSONATE_TARGETS = [
+    "chrome120", "chrome123", "chrome124",
+    "edge101", "edge99",
+    "safari15_5", "safari17_0",
+]
+
+
+class TLSClient:
+    """
+    基于 curl_cffi 的 TLS 指纹伪装 HTTP 客户端
+    能够模拟真实浏览器的 JA3 指纹，绕过 TLS 指纹检测
+    """
+
+    def __init__(self, impersonate: Optional[str] = None):
+        cfg = get_config()
+        self._impersonate = impersonate or cfg.anti_detect.tls.impersonate
+        self._timeout = cfg.crawler.request_timeout
+        self._session = None
+        self._request_count = 0
+        self._max_per_session = cfg.crawler.session.max_requests_per_session
+        self._created_at = None
+
+    async def _ensure_session(self):
+        if self._session is None or self._should_rotate():
+            await self._rotate_session()
+
+    def _should_rotate(self) -> bool:
+        if self._request_count >= self._max_per_session:
+            return True
+        cfg = get_config().crawler.session
+        if self._created_at and (time.time() - self._created_at) > cfg.rotate_interval:
+            return True
+        return False
+
+    async def _rotate_session(self):
+        await self.close()
+        try:
+            from curl_cffi.requests import AsyncSession
+            target = random.choice(IMPERSONATE_TARGETS) if self._impersonate == "random" else self._impersonate
+            self._session = AsyncSession(impersonate=target, timeout=self._timeout)
+            self._request_count = 0
+            self._created_at = time.time()
+            logger.debug(f"TLS 会话已创建，伪装: {target}")
+        except ImportError:
+            logger.warning("curl_cffi 未安装，回退到 aiohttp（无 TLS 指纹伪装）")
+            self._session = None
+            raise
+
+    async def get(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        proxy: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        await self._ensure_session()
+        self._request_count += 1
+
+        full_url = f"{url}?{urlencode(params)}" if params else url
+        logger.info(f"[tls] GET {full_url}")
+
+        start = time.time()
+        try:
+            resp = await self._session.get(
+                url,
+                params=params,
+                headers=headers,
+                proxy=proxy,
+                allow_redirects=True,
+            )
+            elapsed = time.time() - start
+
+            if resp.status_code == 200:
+                try:
+                    data = resp.json()
+                except Exception:
+                    data = resp.text
+                resp_summary = str(data)[:300] if data else '(empty)'
+                logger.info(f"[tls] 200 OK {elapsed:.2f}s {full_url} resp={resp_summary}")
+                return {
+                    "success": True,
+                    "status": resp.status_code,
+                    "data": data,
+                    "elapsed": elapsed,
+                    "headers": dict(resp.headers),
+                }
+
+            elif resp.status_code == 429:
+                logger.warning(f"[tls] 429 RateLimit {elapsed:.2f}s {full_url}")
+                retry_after = int(resp.headers.get("Retry-After", 60))
+                raise RateLimitError(retry_after=retry_after)
+
+            elif resp.status_code == 403:
+                logger.warning(f"[tls] 403 Forbidden {elapsed:.2f}s {full_url}")
+                raise BlockedError(f"403 Forbidden: {url}")
+
+            elif resp.status_code == 503:
+                text = resp.text
+                logger.warning(f"[tls] 503 {elapsed:.2f}s {full_url}")
+                if "captcha" in text.lower() or "验证码" in text:
+                    raise CaptchaError("触发验证码")
+                raise NetworkError(f"503 Service Unavailable: {url}")
+
+            else:
+                logger.warning(f"[tls] HTTP {resp.status_code} {elapsed:.2f}s {full_url}")
+                raise NetworkError(f"HTTP {resp.status_code}: {url}")
+
+        except (RateLimitError, BlockedError, CaptchaError):
+            raise
+        except Exception as e:
+            elapsed = time.time() - start
+            if "curl_cffi" in str(type(e).__module__):
+                raise NetworkError(f"TLS 请求失败: {e}") from e
+            raise
+
+    async def close(self):
+        if self._session:
+            try:
+                await self._session.close()
+            except Exception:
+                pass
+            self._session = None
+            self._created_at = None
+
+    async def __aenter__(self):
+        await self._ensure_session()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close()

docker/crawler/src/core/alert_manager.py

@@ -0,0 +1,183 @@
+import json
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+import aiohttp
+from loguru import logger
+
+from src.core.config import get_config
+from src.storage.database import Database
+
+
+class CrontabAlertManager:
+    def __init__(self, db: Optional[Database] = None):
+        self._cfg = get_config().alert
+        self._db = db or Database()
+
+    @staticmethod
+    def build_dedupe_key(crontab_id: int, alert_type: str) -> str:
+        return f"crontab:{int(crontab_id)}:{alert_type}"
+
+    @staticmethod
+    def _pick_http_status(severe_http_statuses: list[int]) -> int:
+        statuses = [int(item) for item in severe_http_statuses if item is not None]
+        if not statuses:
+            return 0
+        return sorted(statuses)[0]
+
+    async def record_crawler_result(
+        self,
+        action: str,
+        crontab_id: int,
+        crontab_log_id: int,
+        task_result: Dict[str, Any],
+        output: str,
+    ):
+        await self._db.ensure_alert_tables()
+        if action == "error_report":
+            return
+
+        policy = task_result.get("policy", "saved_required")
+        candidate_count = int(task_result.get("candidate_count") or 0)
+        saved_count = int(task_result.get("saved_count") or 0)
+        http_error_count = int(task_result.get("http_error_count") or 0)
+        severe_http_statuses = list(task_result.get("severe_http_statuses") or [])
+        execution_success = bool(task_result.get("execution_success", True))
+        success = bool(task_result.get("success"))
+        error_message = str(task_result.get("error_message") or "")
+        summary_text = str(task_result.get("summary_text") or "")
+
+        alert_type = ""
+        http_status = 0
+        if http_error_count > 0:
+            alert_type = "http_error"
+            http_status = self._pick_http_status(severe_http_statuses)
+        elif not execution_success or (not success and error_message):
+            alert_type = "task_exception"
+        elif policy == "saved_required" and saved_count <= 0:
+            alert_type = "no_data"
+        elif policy == "saved_if_candidates" and candidate_count > 0 and saved_count <= 0:
+            alert_type = "no_data"
+
+        if not alert_type:
+            await self._db.resolve_crontab_alerts(crontab_id)
+            return
+
+        detail = {
+            "summary_text": summary_text[:1000],
+            "candidate_count": candidate_count,
+            "saved_count": saved_count,
+            "http_error_count": http_error_count,
+            "severe_http_statuses": severe_http_statuses,
+            "error_message": error_message[:1000],
+            "output_excerpt": output[:2000],
+        }
+        await self._db.upsert_crontab_alert(
+            crontab_id=crontab_id,
+            crontab_log_id=crontab_log_id,
+            task_name=str(task_result.get("task_name") or action),
+            command=str(task_result.get("command") or "crawler"),
+            params=str(task_result.get("params") or action),
+            alert_type=alert_type,
+            http_status=http_status,
+            detail=detail,
+            dedupe_key=self.build_dedupe_key(crontab_id, alert_type),
+        )
+
+    async def dispatch_open_alerts(self, limit: int = 50) -> Dict[str, Any]:
+        await self._db.ensure_alert_tables()
+        alerts = await self._db.get_dispatchable_crontab_alerts(self._cfg.cooldown_seconds, limit=limit)
+        if not alerts:
+            return {
+                "success": True,
+                "count": 0,
+                "candidate_count": 0,
+                "saved_count": 0,
+                "summary_text": "无待发送告警",
+            }
+
+        if not self._cfg.dispatch_enabled or not self._cfg.wecom_webhook:
+            logger.warning(f"[alert] 企业微信告警未启用或 webhook 未配置，当前待发送 {len(alerts)} 条")
+            return {
+                "success": True,
+                "count": 0,
+                "candidate_count": len(alerts),
+                "saved_count": 0,
+                "summary_text": f"企业微信告警未启用或 webhook 未配置，待发送 {len(alerts)} 条",
+            }
+
+        sent = 0
+        failed = 0
+        for alert in alerts:
+            ok = await self._send_wecom_alert(alert)
+            if ok:
+                await self._db.mark_crontab_alert_notified(int(alert["id"]))
+                sent += 1
+            else:
+                failed += 1
+
+        return {
+            "success": failed == 0,
+            "count": sent,
+            "candidate_count": len(alerts),
+            "saved_count": sent,
+            "failed": failed,
+            "summary_text": f"企业微信告警发送完成：成功 {sent} 条，失败 {failed} 条，待发送 {len(alerts)} 条",
+        }
+
+    async def _send_wecom_alert(self, alert: Dict[str, Any]) -> bool:
+        message = self._build_markdown(alert)
+        payload = {
+            "msgtype": "markdown",
+            "markdown": {"content": message},
+        }
+        timeout = aiohttp.ClientTimeout(total=max(3, int(self._cfg.request_timeout)))
+        try:
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.post(self._cfg.wecom_webhook, json=payload) as resp:
+                    body = await resp.text()
+                    if resp.status != 200:
+                        logger.warning(f"[alert] 企业微信发送失败 HTTP {resp.status}: {body[:300]}")
+                        return False
+                    data = json.loads(body)
+                    if int(data.get("errcode", -1)) != 0:
+                        logger.warning(f"[alert] 企业微信返回失败: {body[:300]}")
+                        return False
+                    return True
+        except Exception as e:
+            logger.warning(f"[alert] 企业微信发送异常: {e}")
+            return False
+
+    @staticmethod
+    def _build_markdown(alert: Dict[str, Any]) -> str:
+        detail_raw = alert.get("detail") or ""
+        try:
+            detail = json.loads(detail_raw) if isinstance(detail_raw, str) else (detail_raw or {})
+        except Exception:
+            detail = {"summary_text": str(detail_raw)}
+
+        def fmt_ts(value: Any) -> str:
+            try:
+                ts = int(value or 0)
+                if ts <= 0:
+                    return "-"
+                return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
+            except Exception:
+                return "-"
+
+        severe = detail.get("severe_http_statuses") or []
+        http_text = "/".join(str(item) for item in severe) if severe else (str(alert.get("http_status") or "-"))
+        summary = str(detail.get("summary_text") or detail.get("error_message") or "无摘要").replace("\n", " ")
+
+        return "\n".join([
+            "### Sport Era 定时任务告警",
+            f"> 任务：`{alert.get('task_name', '')}` / #{alert.get('crontab_id', 0)}",
+            f"> 命令：`{alert.get('command', '')} {alert.get('params', '')}`",
+            f"> 规则：`{alert.get('expression') or '-'}`",
+            f"> 类型：`{alert.get('alert_type', '')}`",
+            f"> HTTP：`{http_text}`",
+            f"> 保存：`{detail.get('saved_count', 0)}` / 候选：`{detail.get('candidate_count', 0)}` / 请求错误：`{detail.get('http_error_count', 0)}`",
+            f"> 首次发生：`{fmt_ts(alert.get('first_seen_at'))}`",
+            f"> 最近发生：`{fmt_ts(alert.get('last_seen_at'))}`",
+            f"> 摘要：{summary[:1000]}",
+        ])

docker/crawler/src/core/config.py

@@ -0,0 +1,266 @@
+"""
+配置加载模块
+"""
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Any
+import yaml
+from pydantic import BaseModel, Field
+
+
+class DatabaseConfig(BaseModel):
+    host: str = "localhost"
+    port: int = 3306
+    database: str = ""
+    username: str = ""
+    password: str = ""
+    charset: str = "utf8mb4"
+    prefix: str = "la_"
+    pool_size: int = 10
+
+
+class RedisConfig(BaseModel):
+    host: str = "localhost"
+    port: int = 6379
+    password: str = ""
+    db: int = 0
+
+
+class DelayConfig(BaseModel):
+    base: float = 3.0
+    min: float = 2.0
+    max: float = 8.0
+    jitter: bool = True
+
+
+class SessionConfig(BaseModel):
+    rotate_interval: int = 300
+    max_requests_per_session: int = 50
+
+
+class CrawlerConfig(BaseModel):
+    mode: str = "hybrid"
+    max_concurrent: int = 3
+    request_timeout: int = 30
+    max_retries: int = 3
+    retry_backoff: float = 2.0
+    delay: DelayConfig = Field(default_factory=DelayConfig)
+    session: SessionConfig = Field(default_factory=SessionConfig)
+
+
+class FingerprintConfig(BaseModel):
+    pool_size: int = 50
+    min_success_rate: float = 0.6
+    rotation_interval: int = 300
+    cooldown: int = 60
+
+
+class TLSConfig(BaseModel):
+    enabled: bool = True
+    impersonate: str = "chrome124"
+
+
+class ViewportConfig(BaseModel):
+    width: int = 1920
+    height: int = 1080
+
+
+class AntiFeaturesConfig(BaseModel):
+    remove_webdriver: bool = True
+    fake_plugins: bool = True
+    fake_languages: bool = True
+    fake_chrome_runtime: bool = True
+    disable_webrtc: bool = True
+    random_viewport: bool = True
+    human_mouse: bool = True
+    random_scroll: bool = True
+
+
+class PlaywrightAntiConfig(BaseModel):
+    browser: str = "chromium"
+    headless: bool = True
+    stealth: bool = True
+    viewport: ViewportConfig = Field(default_factory=ViewportConfig)
+    anti_features: AntiFeaturesConfig = Field(default_factory=AntiFeaturesConfig)
+
+
+class ProxyConfig(BaseModel):
+    enabled: bool = False
+    rotate: bool = True
+    providers: List[str] = Field(default_factory=list)
+    test_url: str = "https://httpbin.org/ip"
+    max_failures: int = 3
+
+
+class AntiDetectConfig(BaseModel):
+    fingerprint: FingerprintConfig = Field(default_factory=FingerprintConfig)
+    tls: TLSConfig = Field(default_factory=TLSConfig)
+    playwright: PlaywrightAntiConfig = Field(default_factory=PlaywrightAntiConfig)
+    proxy: ProxyConfig = Field(default_factory=ProxyConfig)
+
+
+class APIEndpoints(BaseModel):
+    standings: str = "/soccer/biz/data/standing"
+    schedule: str = "/soccer/biz/data/schedule"
+    match_detail: str = "/soccer/biz/data/match_detail"
+    match_menu: str = "/api/v2/config/match_menu"
+    team_info: str = "/soccer/biz/data/team_info"
+    player_stats: str = "/soccer/biz/data/player_stats"
+    live_matches: str = "/soccer/biz/data/live"
+    match_situation: str = "/mobile/match/situation"
+    match_lineup: str = "/mobile/match/lineup"
+    news: str = "/api/v2/article/list"
+
+
+class DefaultParams(BaseModel):
+    app: str = "dqd"
+    platform: str = "www"
+    version: str = "0"
+    language: str = "zh-cn"
+
+
+class TargetLeague(BaseModel):
+    league: str
+    league_code: str = ""
+    season_id: int
+    priority: int = 100
+    active: bool = True
+
+
+class DongqiudiConfig(BaseModel):
+    base_url: str = "https://sport-data.dongqiudi.com"
+    web_url: str = "https://www.dongqiudi.com"
+    api: APIEndpoints = Field(default_factory=APIEndpoints)
+    default_params: DefaultParams = Field(default_factory=DefaultParams)
+    targets: List[TargetLeague] = Field(default_factory=list)
+
+
+class LotterySource(BaseModel):
+    name: str
+    code: str
+    category_id: int
+    url: str
+
+
+class LotteryConfig(BaseModel):
+    sources: List[LotterySource] = Field(default_factory=list)
+
+
+class SchedulerConfig(BaseModel):
+    standings_cron: str = "0 2 * * *"
+    schedule_cron: str = "0 */6 * * *"
+    live_cron: str = "*/5 * * * *"
+    news_cron: str = "*/30 * * * *"
+    content_cron: str = "*/10 * * * *"
+    video_cron: str = "*/30 * * * *"
+    csl_match_cron: str = "0 */1 * * *"
+    nba_match_cron: str = "0 */1 * * *"
+    cba_match_cron: str = "0 */1 * * *"
+    epl_match_cron: str = "0 */1 * * *"
+    bundesliga_match_cron: str = "0 */1 * * *"
+    laliga_match_cron: str = "0 */1 * * *"
+    seriea_match_cron: str = "0 */1 * * *"
+    ligue1_match_cron: str = "0 */1 * * *"
+    ucl_match_cron: str = "0 */1 * * *"
+    uel_match_cron: str = "0 */1 * * *"
+    tennis_match_cron: str = "0 */1 * * *"
+    esports_match_cron: str = "0 */1 * * *"
+    sports_match_cron: str = "0 */1 * * *"
+    league_news_cron: str = "*/30 * * * *"
+    article_content_cron: str = "*/1 * * * *"
+
+
+class LoggingConfig(BaseModel):
+    level: str = "INFO"
+    file: str = "logs/crawler.log"
+    max_size_mb: int = 50
+    backup_count: int = 5
+    format: str = "{time:YYYY-MM-DD HH:mm:ss} | {level:<8} | {name}:{function}:{line} - {message}"
+
+
+class AlertConfig(BaseModel):
+    dispatch_enabled: bool = False
+    wecom_webhook: str = ""
+    cooldown_seconds: int = 1800
+    request_timeout: int = 10
+
+
+class AppConfig(BaseModel):
+    database: DatabaseConfig = Field(default_factory=DatabaseConfig)
+    redis: RedisConfig = Field(default_factory=RedisConfig)
+    crawler: CrawlerConfig = Field(default_factory=CrawlerConfig)
+    anti_detect: AntiDetectConfig = Field(default_factory=AntiDetectConfig)
+    dongqiudi: DongqiudiConfig = Field(default_factory=DongqiudiConfig)
+    lottery: LotteryConfig = Field(default_factory=LotteryConfig)
+    scheduler: SchedulerConfig = Field(default_factory=SchedulerConfig)
+    logging: LoggingConfig = Field(default_factory=LoggingConfig)
+    alert: AlertConfig = Field(default_factory=AlertConfig)
+
+
+_config: Optional[AppConfig] = None
+
+
+def load_config(config_path: Optional[str] = None) -> AppConfig:
+    global _config
+
+    if config_path is None:
+        config_path = os.environ.get("DQD_CONFIG_PATH")
+
+    if config_path is None:
+        candidates = [
+            Path(__file__).parent.parent.parent / "config" / "settings.yaml",
+            Path("config/settings.yaml"),
+        ]
+        for p in candidates:
+            if p.exists():
+                config_path = str(p)
+                break
+
+    if config_path and Path(config_path).exists():
+        with open(config_path, "r", encoding="utf-8") as f:
+            raw = yaml.safe_load(f) or {}
+        raw = _apply_env_overrides(raw)
+        _config = AppConfig(**raw)
+    else:
+        _config = AppConfig(**_apply_env_overrides({}))
+
+    return _config
+
+
+def _apply_env_overrides(raw: Dict[str, Any]) -> Dict[str, Any]:
+    mapping = {
+        "DQD_DB_HOST": ("database", "host", str),
+        "DQD_DB_PORT": ("database", "port", int),
+        "DQD_DB_NAME": ("database", "database", str),
+        "DQD_DB_USER": ("database", "username", str),
+        "DQD_DB_PASSWORD": ("database", "password", str),
+        "DQD_DB_PREFIX": ("database", "prefix", str),
+        "DQD_DB_CHARSET": ("database", "charset", str),
+        "DQD_DB_POOL_SIZE": ("database", "pool_size", int),
+        "DQD_REDIS_HOST": ("redis", "host", str),
+        "DQD_REDIS_PORT": ("redis", "port", int),
+        "DQD_REDIS_PASSWORD": ("redis", "password", str),
+        "DQD_REDIS_DB": ("redis", "db", int),
+        "DQD_LOG_FILE": ("logging", "file", str),
+        "DQD_LOG_LEVEL": ("logging", "level", str),
+    }
+
+    for env_name, (section, key, caster) in mapping.items():
+        if env_name not in os.environ:
+            continue
+        value = os.environ.get(env_name, "")
+        try:
+            cast_value = caster(value)
+        except (TypeError, ValueError):
+            continue
+        raw.setdefault(section, {})
+        raw[section][key] = cast_value
+
+    return raw
+
+
+def get_config() -> AppConfig:
+    global _config
+    if _config is None:
+        _config = load_config()
+    return _config

docker/crawler/src/core/error_collector.py

@@ -0,0 +1,93 @@
+"""
+全局错误收集器
+- 收集所有HTTP请求错误，异步写入数据库
+- 不阻塞主流程，写入失败只打日志
+"""
+import asyncio
+from typing import Optional
+
+from loguru import logger
+
+
+class ErrorCollector:
+    _instance: Optional["ErrorCollector"] = None
+    _task_name: str = ""
+
+    def __init__(self):
+        self._db = None
+        self.reset_run_stats()
+
+    @classmethod
+    def get(cls) -> "ErrorCollector":
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def set_task_name(self, name: str):
+        self._task_name = name
+
+    def reset_run_stats(self):
+        self._http_error_count = 0
+        self._severe_http_statuses = set()
+        self._recent_errors = []
+
+    def get_run_stats(self) -> dict:
+        return {
+            "http_error_count": self._http_error_count,
+            "severe_http_statuses": sorted(self._severe_http_statuses),
+            "recent_errors": list(self._recent_errors),
+        }
+
+    def _record_run_error(self, request_url: str, http_status: int, error_type: str, error_message: str):
+        should_count = http_status == 404 or http_status == 0 or http_status >= 500
+        if should_count:
+            self._http_error_count += 1
+            self._severe_http_statuses.add(int(http_status))
+
+        if len(self._recent_errors) < 10:
+            self._recent_errors.append({
+                "request_url": request_url[:300],
+                "http_status": int(http_status),
+                "error_type": error_type[:100],
+                "error_message": error_message[:500],
+            })
+
+    async def _get_db(self):
+        if self._db is None:
+            from src.storage.database import Database
+            self._db = Database()
+            await self._db.connect()
+        return self._db
+
+    async def report(
+        self,
+        request_url: str,
+        request_method: str = "GET",
+        request_params: str = "",
+        http_status: int = 0,
+        error_type: str = "",
+        error_message: str = "",
+        response_body: str = "",
+        channel: str = "",
+    ):
+        try:
+            self._record_run_error(request_url, http_status, error_type, error_message)
+            db = await self._get_db()
+            await db.insert_error_log(
+                task_name=self._task_name,
+                request_url=request_url,
+                request_method=request_method,
+                request_params=request_params,
+                http_status=http_status,
+                error_type=error_type,
+                error_message=error_message,
+                response_body=response_body,
+                channel=channel,
+            )
+        except Exception as e:
+            logger.debug(f"[error_collector] 写入错误日志失败: {e}")
+
+    async def close(self):
+        if self._db:
+            await self._db.close()
+            self._db = None

docker/crawler/src/core/exceptions.py

@@ -0,0 +1,45 @@
+"""
+异常定义
+"""
+
+
+class CrawlerError(Exception):
+    pass
+
+
+class RateLimitError(CrawlerError):
+    def __init__(self, message="频率限制", retry_after: int = 60):
+        self.retry_after = retry_after
+        super().__init__(message)
+
+
+class BlockedError(CrawlerError):
+    pass
+
+
+class CaptchaError(CrawlerError):
+    pass
+
+
+class NetworkError(CrawlerError):
+    pass
+
+
+class ParsingError(CrawlerError):
+    pass
+
+
+class FingerprintError(CrawlerError):
+    pass
+
+
+class ProxyError(CrawlerError):
+    pass
+
+
+class AntiDetectError(CrawlerError):
+    pass
+
+
+class SessionExpiredError(CrawlerError):
+    pass

docker/crawler/src/core/logger.py

@@ -0,0 +1,36 @@
+"""
+日志模块 - 基于 loguru
+"""
+import sys
+from pathlib import Path
+from loguru import logger
+
+from src.core.config import get_config
+
+
+def setup_logger():
+    cfg = get_config().logging
+
+    logger.remove()
+
+    logger.add(
+        sys.stderr,
+        level=cfg.level,
+        format=cfg.format,
+        colorize=True,
+    )
+
+    log_path = Path(cfg.file)
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+
+    logger.add(
+        str(log_path),
+        level=cfg.level,
+        format=cfg.format,
+        rotation=f"{cfg.max_size_mb} MB",
+        retention=cfg.backup_count,
+        encoding="utf-8",
+        enqueue=True,
+    )
+
+    return logger

docker/crawler/src/engine/api_client.py

@@ -0,0 +1,275 @@
+"""
+懂球帝 API 客户端
+- 支持 aiohttp / curl_cffi 双通道
+- 自动指纹轮换、频率控制、重试
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, Optional
+from urllib.parse import urlencode
+
+import aiohttp
+from loguru import logger
+from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
+
+from src.core.config import get_config
+from src.core.exceptions import (
+    NetworkError, RateLimitError, BlockedError, CaptchaError, ParsingError,
+)
+from src.anti_detect.fingerprint import FingerprintPool
+
+
+class APIClient:
+    """懂球帝 API 客户端 (aiohttp 通道)"""
+
+    def __init__(self, fingerprint_pool: Optional[FingerprintPool] = None):
+        self._cfg = get_config()
+        self._fp_pool = fingerprint_pool
+        self._session: Optional[aiohttp.ClientSession] = None
+        self._request_count = 0
+        self._created_at: Optional[float] = None
+        self._stats = {
+            "total": 0, "success": 0, "fail": 0,
+            "rate_limit": 0, "blocked": 0, "total_time": 0.0,
+        }
+
+    async def initialize(self):
+        if self._session is None or self._should_rotate():
+            await self._new_session()
+
+    async def _new_session(self):
+        await self.close()
+        timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
+        self._session = aiohttp.ClientSession(timeout=timeout)
+        self._request_count = 0
+        self._created_at = time.time()
+
+    def _should_rotate(self) -> bool:
+        scfg = self._cfg.crawler.session
+        if self._request_count >= scfg.max_requests_per_session:
+            return True
+        if self._created_at and (time.time() - self._created_at) > scfg.rotate_interval:
+            return True
+        return False
+
+    async def close(self):
+        if self._session:
+            await self._session.close()
+            self._session = None
+
+    async def _get_headers(self) -> tuple:
+        """获取请求头，返回 (headers, fp_id)"""
+        if self._fp_pool:
+            fp = await self._fp_pool.acquire()
+            return fp.get("http_headers", {}), fp.get("id")
+
+        return {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Referer": "https://www.dongqiudi.com/",
+            "Origin": "https://www.dongqiudi.com",
+        }, None
+
+    async def request(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        proxy: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        await self.initialize()
+        headers, fp_id = await self._get_headers()
+        self._request_count += 1
+        self._stats["total"] += 1
+        start = time.time()
+
+        full_url = f"{url}?{urlencode(params)}" if params else url
+        logger.info(f"[api] GET {full_url}")
+
+        try:
+            async with self._session.get(
+                url, params=params, headers=headers, proxy=proxy,
+                allow_redirects=True, ssl=False,
+            ) as resp:
+                elapsed = time.time() - start
+                self._stats["total_time"] += elapsed
+                return await self._handle_response(resp, url, fp_id, elapsed, params)
+
+        except (RateLimitError, BlockedError, CaptchaError):
+            self._stats["fail"] += 1
+            raise
+        except aiohttp.ClientError as e:
+            self._stats["fail"] += 1
+            if fp_id and self._fp_pool:
+                await self._fp_pool.mark_failure(fp_id, "network")
+            raise NetworkError(f"网络错误: {e}") from e
+        except Exception as e:
+            self._stats["fail"] += 1
+            raise NetworkError(f"请求异常: {e}") from e
+
+    async def _handle_response(
+        self, resp: aiohttp.ClientResponse, url: str, fp_id: Optional[str], elapsed: float,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        status = resp.status
+        full_url = f"{url}?{urlencode(params)}" if params else url
+
+        if status == 200:
+            try:
+                data = await resp.json(content_type=None)
+            except Exception:
+                text = await resp.text()
+                raise ParsingError(f"JSON 解析失败，原始响应: {text[:500]}")
+
+            resp_summary = str(data)[:300] if data else '(empty)'
+            logger.info(f"[api] 200 OK {elapsed:.2f}s {full_url} resp={resp_summary}")
+            self._stats["success"] += 1
+            if fp_id and self._fp_pool:
+                await self._fp_pool.mark_success(fp_id)
+            return data
+
+        elif status == 429:
+            logger.warning(f"[api] 429 RateLimit {elapsed:.2f}s {full_url}")
+            self._stats["rate_limit"] += 1
+            retry_after = int(resp.headers.get("Retry-After", 60))
+            if fp_id and self._fp_pool:
+                await self._fp_pool.mark_failure(fp_id, "rate_limit")
+            raise RateLimitError(retry_after=retry_after)
+
+        elif status == 403:
+            logger.warning(f"[api] 403 Forbidden {elapsed:.2f}s {full_url}")
+            self._stats["blocked"] += 1
+            if fp_id and self._fp_pool:
+                await self._fp_pool.mark_failure(fp_id, "ip_blocked")
+            raise BlockedError(f"403: {url}")
+
+        elif status == 503:
+            text = await resp.text()
+            logger.warning(f"[api] 503 {elapsed:.2f}s {full_url}")
+            if "captcha" in text.lower() or "验证码" in text:
+                if fp_id and self._fp_pool:
+                    await self._fp_pool.mark_failure(fp_id, "captcha")
+                raise CaptchaError("验证码")
+            raise NetworkError(f"503: {url}")
+
+        else:
+            logger.warning(f"[api] HTTP {status} {elapsed:.2f}s {full_url}")
+            raise NetworkError(f"HTTP {status}: {url}")
+
+    # ── 懂球帝专用 API 方法 ──
+
+    def _default_params(self, extra: Optional[Dict] = None) -> Dict[str, Any]:
+        dp = self._cfg.dongqiudi.default_params
+        params = {"app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
+        if extra:
+            params.update(extra)
+        return params
+
+    async def get_standings(self, season_id: int) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.standings
+        params = self._default_params({"season_id": season_id})
+        return await self.request(url, params)
+
+    async def get_schedule(self, season_id: int, round_number: Optional[int] = None) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.schedule
+        extra = {"season_id": season_id}
+        if round_number:
+            extra["round"] = round_number
+        return await self.request(url, self._default_params(extra))
+
+    async def get_match_detail(self, match_id: int) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.match_detail
+        return await self.request(url, self._default_params({"match_id": match_id}))
+
+    async def get_match_menu(self) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.web_url + self._cfg.dongqiudi.api.match_menu
+        params = {"mark": "gif", "platform": "www", "version": "713"}
+        return await self.request(url, params)
+
+    async def get_live_matches(self) -> Dict[str, Any]:
+        """从各联赛 tab API 汇总当前比赛数据"""
+        all_matches = []
+        league_ids = set()
+        for t in self._cfg.dongqiudi.targets:
+            if not t.active:
+                continue
+            lid = self._get_league_id(t.league)
+            if lid and lid not in league_ids:
+                league_ids.add(lid)
+                try:
+                    url = f"https://api.dongqiudi.com/data/tab/league/new/{lid}"
+                    raw = await self.request(url, None)
+                    items = raw.get("list", []) if isinstance(raw, dict) else []
+                    all_matches.extend(items)
+                except Exception as e:
+                    logger.warning(f"获取联赛 {t.league} 实时数据失败: {e}")
+        return {"data": {"matches": all_matches}}
+
+    _LEAGUE_ID_MAP = {
+        "中超": 43, "中甲": 129, "亚冠": 226, "英超": 4, "西甲": 3,
+        "德甲": 5, "意甲": 9, "法甲": 12, "欧冠": 6, "欧联": 14,
+    }
+
+    def _get_league_id(self, league_name: str) -> Optional[int]:
+        return self._LEAGUE_ID_MAP.get(league_name)
+
+    async def get_news(self, tab_id: int = 1, size: int = 30) -> Dict[str, Any]:
+        """通过 tabs API 获取资讯列表"""
+        url = f"{self._cfg.dongqiudi.web_url}/api/app/tabs/web/{tab_id}.json"
+        params = {"size": size} if size != 20 else None
+        return await self.request(url, params)
+
+    async def get_video_list(self, tab_id: int = 233) -> Dict[str, Any]:
+        """抓取视频列表页 HTML，返回 {'html': str, 'tab_id': int}"""
+        await self.initialize()
+        url = f"{self._cfg.dongqiudi.web_url}/videoList/{tab_id}"
+        headers, fp_id = await self._get_headers()
+        headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        self._request_count += 1
+        self._stats["total"] += 1
+        try:
+            async with self._session.get(url, headers=headers, ssl=False) as resp:
+                if resp.status == 200:
+                    html = await resp.text()
+                    self._stats["success"] += 1
+                    return {"html": html, "tab_id": tab_id}
+                raise NetworkError(f"HTTP {resp.status}: {url}")
+        except aiohttp.ClientError as e:
+            self._stats["fail"] += 1
+            raise NetworkError(f"网络错误: {e}") from e
+
+    async def get_article_detail(self, article_id: int) -> Dict[str, Any]:
+        """抓取文章详情页 HTML，返回 {'html': str}"""
+        await self.initialize()
+        url = f"{self._cfg.dongqiudi.web_url}/articles/{article_id}.html"
+        headers, fp_id = await self._get_headers()
+        headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        self._request_count += 1
+        self._stats["total"] += 1
+        try:
+            async with self._session.get(url, headers=headers, ssl=False) as resp:
+                if resp.status == 200:
+                    html = await resp.text()
+                    self._stats["success"] += 1
+                    return {"html": html, "article_id": article_id}
+                raise NetworkError(f"HTTP {resp.status}: {url}")
+        except aiohttp.ClientError as e:
+            self._stats["fail"] += 1
+            raise NetworkError(f"网络错误: {e}") from e
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        total = self._stats["total"]
+        return {
+            **self._stats,
+            "success_rate": self._stats["success"] / total if total else 1.0,
+            "avg_time": self._stats["total_time"] / total if total else 0.0,
+        }
+
+    async def __aenter__(self):
+        await self.initialize()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close()

docker/crawler/src/engine/hybrid_engine.py

@@ -0,0 +1,497 @@
+"""
+混合爬虫引擎
+- API 优先 → TLS 伪装降级 → Playwright 降级
+- 智能选择最优通道、自动重试、频率控制
+"""
+import asyncio
+import json
+import random
+import re
+import time
+from enum import Enum
+from typing import Dict, Any, Optional, List
+
+import aiohttp
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.error_collector import ErrorCollector
+from src.core.exceptions import (
+    CrawlerError, RateLimitError, BlockedError, CaptchaError, NetworkError,
+)
+from src.anti_detect.fingerprint import FingerprintPool
+from src.anti_detect.proxy_manager import ProxyManager
+from src.engine.api_client import APIClient
+
+
+class Channel(Enum):
+    API = "api"
+    TLS = "tls"
+    PLAYWRIGHT = "playwright"
+
+
+class HybridEngine:
+    """混合爬虫引擎"""
+
+    def __init__(self):
+        self._cfg = get_config()
+        self._mode = self._cfg.crawler.mode
+
+        # 组件
+        self._fp_pool = FingerprintPool()
+        self._proxy_mgr = ProxyManager()
+        self._api_client: Optional[APIClient] = None
+        self._tls_client = None
+        self._browser = None
+
+        # 通道统计（用于智能模式选择）
+        self._channel_stats: Dict[str, Dict[str, int]] = {
+            ch.value: {"success": 0, "fail": 0} for ch in Channel
+        }
+
+        self._initialized = False
+
+    async def initialize(self):
+        if self._initialized:
+            return
+
+        await self._fp_pool.initialize()
+        self._api_client = APIClient(self._fp_pool)
+
+        # TLS 客户端（可选）
+        if self._cfg.anti_detect.tls.enabled:
+            try:
+                from src.anti_detect.tls_client import TLSClient
+                self._tls_client = TLSClient()
+                logger.info("TLS 伪装客户端已就绪")
+            except Exception as e:
+                logger.warning(f"TLS 客户端初始化失败（curl_cffi 可能未安装）: {e}")
+
+        self._initialized = True
+        logger.info(f"混合引擎初始化完成，模式: {self._mode}")
+
+    async def close(self):
+        if self._api_client:
+            await self._api_client.close()
+        if self._tls_client:
+            await self._tls_client.close()
+        if self._browser:
+            await self._browser.stop()
+        self._initialized = False
+
+    # ── 通用请求方法 ──
+
+    async def request(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        max_retries: Optional[int] = None,
+    ) -> Dict[str, Any]:
+        """
+        统一请求入口，根据模式自动选择通道并降级
+        """
+        await self.initialize()
+
+        retries = max_retries or self._cfg.crawler.max_retries
+        channels = self._get_channel_order()
+        last_error = None
+        ec = ErrorCollector.get()
+        full_url = url
+        if params:
+            from urllib.parse import urlencode
+            full_url = f"{url}?{urlencode(params)}"
+        params_str = json.dumps(params, ensure_ascii=False) if params else ""
+
+        for attempt in range(retries):
+            for channel in channels:
+                try:
+                    await self._smart_delay()
+                    proxy = await self._proxy_mgr.get_proxy() if self._proxy_mgr.enabled else None
+
+                    result = await self._dispatch(channel, url, params, proxy)
+                    self._channel_stats[channel.value]["success"] += 1
+                    return result
+
+                except RateLimitError as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    wait = e.retry_after * (1 + random.uniform(0, 0.3))
+                    logger.warning(f"[{channel.value}] 频率限制，等待 {wait:.0f}s")
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=429, error_type="RateLimitError", error_message=str(e), channel=channel.value)
+                    await asyncio.sleep(wait)
+                    last_error = e
+
+                except BlockedError as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    logger.warning(f"[{channel.value}] IP 被封，尝试下一通道")
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=403, error_type="BlockedError", error_message=str(e), channel=channel.value)
+                    last_error = e
+                    continue
+
+                except CaptchaError as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    logger.warning(f"[{channel.value}] 触发验证码，降级到 Playwright")
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=503, error_type="CaptchaError", error_message=str(e), channel=channel.value)
+                    if channel != Channel.PLAYWRIGHT:
+                        continue
+                    last_error = e
+
+                except (NetworkError, CrawlerError) as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    logger.warning(f"[{channel.value}] 请求失败: {e}")
+                    http_code = 0
+                    err_msg = str(e)
+                    if "404" in err_msg:
+                        http_code = 404
+                    elif "500" in err_msg:
+                        http_code = 500
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=http_code, error_type=type(e).__name__, error_message=err_msg, channel=channel.value)
+                    last_error = e
+                    continue
+
+            # 本轮所有通道都失败，退避后重试
+            backoff = self._cfg.crawler.retry_backoff ** attempt * (1 + random.uniform(0, 0.5))
+            logger.info(f"第 {attempt + 1} 轮全部失败，退避 {backoff:.1f}s 后重试")
+            await asyncio.sleep(backoff)
+
+        raise last_error or CrawlerError(f"所有通道和重试均失败: {url}")
+
+    async def _dispatch(
+        self, channel: Channel, url: str, params: Optional[Dict], proxy: Optional[str],
+    ) -> Dict[str, Any]:
+        """分发到具体通道"""
+        if channel == Channel.API:
+            return await self._api_client.request(url, params, proxy)
+
+        elif channel == Channel.TLS:
+            if not self._tls_client:
+                raise NetworkError("TLS 客户端不可用")
+            fp = await self._fp_pool.acquire()
+            headers = fp.get("http_headers", {})
+            result = await self._tls_client.get(url, params=params, headers=headers, proxy=proxy)
+            if result.get("success"):
+                return result["data"]
+            raise NetworkError(result.get("error", "TLS 请求失败"))
+
+        elif channel == Channel.PLAYWRIGHT:
+            if not self._browser:
+                from src.anti_detect.stealth_browser import StealthBrowser
+                self._browser = StealthBrowser()
+                await self._browser.start()
+
+            full_url = url
+            if params:
+                from urllib.parse import urlencode
+                full_url = f"{url}?{urlencode(params)}"
+
+            logger.info(f"[playwright] GET {full_url}")
+            result = await self._browser.fetch_page(full_url, extract_json=True)
+            if result.get("success") and result.get("json_data"):
+                resp_summary = str(result["json_data"])[:300]
+                logger.info(f"[playwright] 200 OK {full_url} resp={resp_summary}")
+                return result["json_data"]
+            elif result.get("success"):
+                logger.info(f"[playwright] OK (no json) {full_url}")
+                return result
+            logger.warning(f"[playwright] FAIL {full_url} error={result.get('error','')}")
+            raise NetworkError(result.get("error", "Playwright 请求失败"))
+
+        raise CrawlerError(f"未知通道: {channel}")
+
+    def _get_channel_order(self) -> List[Channel]:
+        """根据模式和历史成功率决定通道优先级"""
+        if self._mode == "api_only":
+            return [Channel.API]
+        elif self._mode == "playwright_only":
+            return [Channel.PLAYWRIGHT]
+        elif self._mode == "hybrid":
+            order = [Channel.API]
+            if self._tls_client:
+                order.append(Channel.TLS)
+            order.append(Channel.PLAYWRIGHT)
+            return order
+        elif self._mode == "smart":
+            return self._smart_channel_order()
+        return [Channel.API, Channel.PLAYWRIGHT]
+
+    def _smart_channel_order(self) -> List[Channel]:
+        """智能排序：按成功率排序"""
+        def _rate(ch: Channel) -> float:
+            s = self._channel_stats[ch.value]
+            total = s["success"] + s["fail"]
+            return s["success"] / total if total > 5 else 0.5
+
+        candidates = [Channel.API]
+        if self._tls_client:
+            candidates.append(Channel.TLS)
+        candidates.append(Channel.PLAYWRIGHT)
+
+        candidates.sort(key=_rate, reverse=True)
+        return candidates
+
+    async def _smart_delay(self):
+        """高斯分布随机延迟"""
+        dcfg = self._cfg.crawler.delay
+        delay = random.gauss(dcfg.base, (dcfg.max - dcfg.min) / 4)
+        delay = max(dcfg.min, min(dcfg.max, delay))
+        if dcfg.jitter:
+            delay += random.uniform(-0.5, 0.5)
+            delay = max(0.5, delay)
+        await asyncio.sleep(delay)
+
+    # ── 懂球帝专用方法（通过混合引擎转发） ──
+
+    async def get_standings(self, season_id: int) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.standings
+        dp = self._cfg.dongqiudi.default_params
+        params = {"season_id": season_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
+        return await self.request(url, params)
+
+    async def get_schedule(self, season_id: int, round_number: Optional[int] = None) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.schedule
+        dp = self._cfg.dongqiudi.default_params
+        params = {"season_id": season_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
+        if round_number:
+            params["round"] = round_number
+        return await self.request(url, params)
+
+    async def get_match_detail(self, match_id: int) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.match_detail
+        dp = self._cfg.dongqiudi.default_params
+        params = {"match_id": match_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
+        return await self.request(url, params)
+
+    async def get_match_situation(self, match_id: int) -> Dict[str, Any]:
+        """获取比赛实况数据（situation API）"""
+        url = f"https://api.dongqiudi.com{self._cfg.dongqiudi.api.match_situation}/{match_id}"
+        return await self.request(url)
+
+    async def get_match_lineup(self, match_id: int) -> Dict[str, Any]:
+        """获取比赛阵容数据（lineup API）"""
+        url = f"https://api.dongqiudi.com{self._cfg.dongqiudi.api.match_lineup}/{match_id}"
+        return await self.request(url)
+
+    async def fetch_match_detail_from_page(self, match_id: int) -> Dict[str, Any]:
+        """从懂球帝移动端页面抓取比赛详情(SSR数据)"""
+        page_url = f"http://m.dongqiudi.com/matchDetail/{match_id}/lotteryOddsNew?cmp_type=soccer"
+        logger.info(f"[page] GET {page_url}")
+        start = time.time()
+        ec = ErrorCollector.get()
+
+        timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
+        headers = {
+            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) "
+                          "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+        }
+        try:
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(page_url, headers=headers, ssl=False) as resp:
+                    elapsed = time.time() - start
+                    if resp.status != 200:
+                        logger.warning(f"[page] HTTP {resp.status} {elapsed:.2f}s {page_url}")
+                        await ec.report(request_url=page_url, http_status=resp.status, error_type=f"HTTP {resp.status}", error_message=f"页面请求失败 HTTP {resp.status}", channel="page")
+                        raise NetworkError(f"页面请求失败 HTTP {resp.status}: {page_url}")
+                    html = await resp.text()
+                    logger.info(f"[page] 200 OK {elapsed:.2f}s {page_url} html_len={len(html)}")
+        except NetworkError:
+            raise
+        except Exception as e:
+            await ec.report(request_url=page_url, http_status=0, error_type=type(e).__name__, error_message=str(e), channel="page")
+            raise NetworkError(f"页面请求异常: {page_url} {e}") from e
+
+        m = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})\s*;', html, re.DOTALL)
+        if not m:
+            await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="页面中未找到 __INITIAL_STATE__", response_body=html[:2000], channel="page")
+            raise NetworkError(f"页面中未找到 __INITIAL_STATE__: {page_url}")
+
+        state = json.loads(m.group(1))
+        header_data = (state.get("matchContentStore", {})
+                            .get("matchDetailHeaderData", {}))
+        if not header_data:
+            await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="matchDetailHeaderData 为空", channel="page")
+            raise NetworkError(f"matchDetailHeaderData 为空: {page_url}")
+
+        match_obj = header_data.get("match", {})
+        info_obj = header_data.get("info", {})
+        logger.info(
+            f"[page] 解析完成: match_id={match_id} "
+            f"status={match_obj.get('status','')} "
+            f"team_A.fs={match_obj.get('team_A',{}).get('fs','')} "
+            f"team_B.fs={match_obj.get('team_B',{}).get('fs','')} "
+            f"events_count={len(info_obj.get('events') or [])}"
+        )
+        return header_data
+
+    async def fetch_match_list_from_page(self, tab_id: int = 5) -> List[Dict[str, Any]]:
+        """从懂球帝移动端赛程页面抓取 matchListStore.matchList（SSR数据）"""
+        page_url = f"http://m.dongqiudi.com/match/{tab_id}"
+        logger.info(f"[page] GET {page_url}")
+        start = time.time()
+        ec = ErrorCollector.get()
+
+        timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
+        headers = {
+            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) "
+                          "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+        }
+        try:
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(page_url, headers=headers, ssl=False) as resp:
+                    elapsed = time.time() - start
+                    if resp.status != 200:
+                        logger.warning(f"[page] HTTP {resp.status} {elapsed:.2f}s {page_url}")
+                        await ec.report(request_url=page_url, http_status=resp.status, error_type=f"HTTP {resp.status}", error_message=f"页面请求失败 HTTP {resp.status}", channel="page")
+                        raise NetworkError(f"页面请求失败 HTTP {resp.status}: {page_url}")
+                    html = await resp.text()
+                    logger.info(f"[page] 200 OK {elapsed:.2f}s {page_url} html_len={len(html)}")
+        except NetworkError:
+            raise
+        except Exception as e:
+            await ec.report(request_url=page_url, http_status=0, error_type=type(e).__name__, error_message=str(e), channel="page")
+            raise NetworkError(f"页面请求异常: {page_url} {e}") from e
+
+        m = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})\s*;', html, re.DOTALL)
+        if not m:
+            await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="页面中未找到 __INITIAL_STATE__", response_body=html[:2000], channel="page")
+            raise NetworkError(f"页面中未找到 __INITIAL_STATE__: {page_url}")
+
+        state = json.loads(m.group(1))
+        match_list = state.get("matchListStore", {}).get("matchList", [])
+        logger.info(f"[page] 解析完成: tab_id={tab_id} matchList={len(match_list)} 条")
+        return match_list
+
+    async def get_schedule_round(self, url: str) -> Dict[str, Any]:
+        """请求具体轮次的赛程URL（含round_id/gameweek参数）"""
+        if 'language=' not in url:
+            url += '&language=zh-cn'
+        return await self.request(url)
+
+    async def get_league_tab_matches(self, api_url: str) -> Dict[str, Any]:
+        """通过联赛 tab API 获取比赛列表"""
+        return await self.request(api_url)
+
+    async def get_live_text(self, match_id: int, num: int = 200) -> Dict[str, Any]:
+        """获取比赛文字直播"""
+        url = f"{self._cfg.dongqiudi.web_url}/api/v4/imserver/baidu/Chatroom/rooms"
+        params = {"match_id": match_id, "num": num}
+        return await self.request(url, params)
+
+    async def get_live_text_page(self, page_url: str) -> Dict[str, Any]:
+        """文字直播翻页请求"""
+        return await self.request(page_url)
+
+    async def get_match_menu(self) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.web_url + self._cfg.dongqiudi.api.match_menu
+        params = {"mark": "gif", "platform": "www", "version": "713"}
+        return await self.request(url, params)
+
+    async def get_live_matches(self) -> Dict[str, Any]:
+        """从各联赛 tab API 汇总当前比赛数据"""
+        all_matches = []
+        league_ids = set()
+        for t in self._cfg.dongqiudi.targets:
+            if not t.active:
+                continue
+            lid = self._LEAGUE_ID_MAP.get(t.league)
+            if lid and lid not in league_ids:
+                league_ids.add(lid)
+                try:
+                    url = f"https://api.dongqiudi.com/data/tab/league/new/{lid}"
+                    raw = await self.request(url, None)
+                    items = raw.get("list", []) if isinstance(raw, dict) else []
+                    all_matches.extend(items)
+                except Exception as e:
+                    logger.warning(f"获取联赛 {t.league} 实时数据失败: {e}")
+        return {"data": {"matches": all_matches}}
+
+    _LEAGUE_ID_MAP = {
+        "中超": 43, "中甲": 129, "亚冠": 226, "英超": 4, "西甲": 3,
+        "德甲": 5, "意甲": 9, "法甲": 12, "欧冠": 6, "欧联": 14,
+    }
+
+    async def get_news(self, tab_id: int = 1, size: int = 30) -> Dict[str, Any]:
+        url = f"{self._cfg.dongqiudi.web_url}/api/app/tabs/web/{tab_id}.json"
+        params = {"size": size} if size != 20 else None
+        return await self.request(url, params)
+
+    async def get_league_news(self, tab_id: int, page: int = 1, after: int = 0) -> Dict[str, Any]:
+        """从懂球帝 iPhone tabs API 获取联赛资讯（支持分页）"""
+        url = f"https://api.dongqiudi.com/app/tabs/iphone/{tab_id}.json"
+        params = {"mark": "gif", "version": "576", "from": f"tab_{tab_id}"}
+        if after:
+            params["after"] = after
+            params["page"] = page
+        return await self.request(url, params)
+
+    async def get_video_list(self, tab_id: int = 233) -> Dict[str, Any]:
+        await self.initialize()
+        return await self._api_client.get_video_list(tab_id)
+
+    async def get_article_detail(self, article_id: int) -> Dict[str, Any]:
+        await self.initialize()
+        return await self._api_client.get_article_detail(article_id)
+
+    async def get_article_html(self, article_id: int) -> str:
+        """请求 m.dongqiudi.com 文章页面，返回 HTML 字符串"""
+        import aiohttp
+        url = f"https://m.dongqiudi.com/article/{article_id}.html"
+        headers = {
+            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
+        }
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
+            async with session.get(url) as resp:
+                if resp.status == 200:
+                    return await resp.text()
+                raise NetworkError(f"HTTP {resp.status}: {url}")
+
+    async def get_lottery_draw_data(self, url: str) -> dict:
+        """请求彩种开奖数据API（返回JSON）"""
+        await self.initialize()
+        import aiohttp
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(url) as resp:
+                if resp.status == 200:
+                    return await resp.json()
+                raise NetworkError(f"HTTP {resp.status}: {url}")
+
+    async def get_lottery_result(self, url: str) -> str:
+        """请求六合彩开奖结果（返回原始文本，自签名证书跳过验证）"""
+        await self.initialize()
+        import aiohttp, ssl
+        ssl_ctx = ssl.create_default_context()
+        ssl_ctx.check_hostname = False
+        ssl_ctx.verify_mode = ssl.CERT_NONE
+        ts = int(time.time() * 1000)
+        full_url = f"{url}?_={ts}"
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(full_url, ssl=ssl_ctx) as resp:
+                if resp.status == 200:
+                    raw = await resp.read()
+                    # 尝试 utf-8 解码，失败则用 gbk
+                    try:
+                        return raw.decode('utf-8')
+                    except UnicodeDecodeError:
+                        return raw.decode('gbk', errors='replace')
+                raise NetworkError(f"HTTP {resp.status}: {full_url}")
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        return {
+            "channels": self._channel_stats,
+            "fingerprint": self._fp_pool.stats,
+            "proxy": self._proxy_mgr.stats,
+            "api_client": self._api_client.stats if self._api_client else {},
+        }
+
+    async def __aenter__(self):
+        await self.initialize()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close()

docker/crawler/src/parser/data_parser.py

@@ -0,0 +1,560 @@
+"""
+懂球帝数据解析器
+- 积分榜、赛程、比赛详情、新闻等数据解析
+- 数据校验和清洗
+"""
+import re
+from datetime import datetime
+from typing import Dict, Any, List, Optional, Tuple
+
+from loguru import logger
+
+from src.core.exceptions import ParsingError
+
+
+class DongqiudiParser:
+    """懂球帝数据解析器"""
+
+    # ── 积分榜 ──
+
+    def parse_standings(self, response: Dict[str, Any], season_id: int) -> List[Dict[str, Any]]:
+        """解析积分榜 API 响应"""
+        if not isinstance(response, dict):
+            raise ParsingError(f"积分榜响应类型错误: {type(response)}")
+
+        content = response.get("content", {})
+        rounds = content.get("rounds", [])
+        if not rounds:
+            logger.warning(f"积分榜无 rounds 数据 (season_id={season_id})")
+            return []
+
+        standings = []
+        for round_data in rounds:
+            rc = round_data.get("content", {})
+            round_info = rc.get("info", {})
+            round_number = round_info.get("round", 0)
+            teams = rc.get("data", [])
+
+            for team in teams:
+                parsed = self._parse_team_standing(team, season_id, round_number)
+                if parsed:
+                    standings.append(parsed)
+
+        logger.info(f"积分榜解析完成: season_id={season_id}, {len(standings)} 条记录")
+        return standings
+
+    def _parse_team_standing(self, team: Dict, season_id: int, round_number: int) -> Optional[Dict[str, Any]]:
+        team_id = team.get("team_id")
+        if not team_id:
+            return None
+
+        deduction_pts, deduction_reason = self._parse_deduction(team.get("instruction", {}))
+        recent = self._parse_recent_form(team.get("recent_record", ""))
+
+        gf = int(team.get("goals_pro", 0))
+        ga = int(team.get("goals_against", 0))
+
+        return {
+            "season_id": season_id,
+            "round_number": round_number,
+            "team_id": team_id,
+            "team_name": team.get("team_name", ""),
+            "team_logo": team.get("team_logo", ""),
+            "rank": int(team.get("rank", 0)),
+            "points": int(team.get("points", 0)),
+            "played": int(team.get("matches_total", 0)),
+            "won": int(team.get("matches_won", 0)),
+            "drawn": int(team.get("matches_draw", 0)),
+            "lost": int(team.get("matches_lost", 0)),
+            "goals_for": gf,
+            "goals_against": ga,
+            "goal_diff": gf - ga,
+            "recent_form": recent,
+            "deduction_points": deduction_pts,
+            "deduction_reason": deduction_reason,
+        }
+
+    # ── 赛程 ──
+
+    def parse_schedule(self, response: Dict[str, Any], season_id: int) -> List[Dict[str, Any]]:
+        """解析赛程 API 响应"""
+        if not isinstance(response, dict):
+            raise ParsingError(f"赛程响应类型错误: {type(response)}")
+
+        content = response.get("content", {})
+        matches_raw = content.get("matches", [])
+        if not matches_raw:
+            logger.warning(f"赛程无 matches 数据 (season_id={season_id})")
+            return []
+
+        matches = []
+        for m in matches_raw:
+            parsed = self._parse_match(m, season_id)
+            if parsed:
+                matches.append(parsed)
+
+        logger.info(f"赛程解析完成: season_id={season_id}, {len(matches)} 场比赛")
+        return matches
+
+    def _parse_match(self, m: Dict, season_id: int) -> Optional[Dict[str, Any]]:
+        match_id = m.get("match_id")
+        if not match_id:
+            return None
+
+        start_play = self._parse_datetime(m.get("start_play"))
+        status = self._normalize_status(m.get("status", ""))
+
+        return {
+            "match_id": match_id,
+            "season_id": season_id,
+            "round_number": m.get("round_id", 0),
+            "round_name": m.get("round_name", ""),
+            "match_date": start_play,
+            "status": status,
+            "home_team_id": m.get("team_A_id"),
+            "home_team_name": m.get("team_A_name", ""),
+            "home_team_logo": m.get("team_A_logo", ""),
+            "away_team_id": m.get("team_B_id"),
+            "away_team_name": m.get("team_B_name", ""),
+            "away_team_logo": m.get("team_B_logo", ""),
+            "home_score": m.get("score_A") if status != "fixture" else None,
+            "away_score": m.get("score_B") if status != "fixture" else None,
+            "halftime_score": m.get("half_score", ""),
+            "venue": m.get("stadium", ""),
+            "competition_id": m.get("competition_id"),
+            "competition_name": m.get("competition_name", ""),
+        }
+
+    # ── 比赛详情 ──
+
+    def parse_match_detail(self, response: Dict[str, Any]) -> Dict[str, Any]:
+        """解析比赛详情"""
+        if not isinstance(response, dict):
+            raise ParsingError(f"比赛详情响应类型错误: {type(response)}")
+
+        data = response.get("data", response)
+        return {
+            "match_id": data.get("match_id"),
+            "status": self._normalize_status(data.get("status", "")),
+            "home_team": data.get("team_A", {}),
+            "away_team": data.get("team_B", {}),
+            "home_score": data.get("score_A"),
+            "away_score": data.get("score_B"),
+            "events": data.get("events", []),
+            "statistics": data.get("statistics", {}),
+            "lineups": data.get("lineups", {}),
+        }
+
+    # ── 比赛菜单 / 联赛列表 ──
+
+    def parse_match_menu(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """解析比赛类型菜单"""
+        if response.get("errCode") != 0:
+            return []
+
+        items = response.get("data", {}).get("list", [])
+        result = []
+        for item in items:
+            result.append({
+                "id": item.get("id"),
+                "label": item.get("label", ""),
+                "type": item.get("type", ""),
+                "sort": item.get("sort", 0),
+                "api": item.get("api", ""),
+            })
+        return result
+
+    # ── 新闻 ──
+
+    def parse_news(self, response: Dict[str, Any], tab_name: str = "") -> List[Dict[str, Any]]:
+        """解析 /api/app/tabs/web/{id}.json 返回的文章列表"""
+        articles = response.get("articles", [])
+        if not articles:
+            return []
+
+        import time
+        now_ts = int(time.time())
+
+        result = []
+        for art in articles:
+            aid = art.get("id")
+            if not aid:
+                continue
+            title = art.get("title", "")
+            if not title:
+                continue
+
+            author_info = art.get("author")
+            if isinstance(author_info, dict):
+                author = author_info.get("name", "")
+            else:
+                author = str(art.get("author_name", "") or "")
+
+            result.append({
+                "id": aid,
+                "title": title,
+                "description": art.get("description", "") or art.get("b_description", ""),
+                "thumb": art.get("thumb", ""),
+                "author_name": author,
+                "published_at": art.get("published_at", ""),
+                "share": art.get("share", ""),
+                "comments_total": art.get("comments_total", 0),
+                "category": tab_name or "news",
+                "is_video": art.get("is_video", False),
+                "sort_timestamp": now_ts,
+            })
+
+        logger.info(f"从tabs API解析到 {len(result)} 篇文章 (tab={tab_name})")
+        return result
+
+    # ── 视频列表 ──
+
+    def parse_video_list(self, response: Dict[str, Any], tab_name: str = "") -> List[Dict[str, Any]]:
+        """从视频列表页 HTML 的 __NUXT__ 中提取视频条目列表"""
+        import json as _json
+        html = response.get("html", "")
+        if not html:
+            return []
+
+        nuxt_m = re.search(r'window\.__NUXT__\s*=\s*(.+?);\s*</script>', html, re.DOTALL)
+        if not nuxt_m:
+            return []
+
+        nuxt_raw = nuxt_m.group(1)
+
+        var_map = self._parse_iife_var_map(nuxt_raw)
+
+        ids = re.findall(r'\bid:(\d{5,})', nuxt_raw)
+        shares = re.findall(r'\bshare:"([^"]*)"', nuxt_raw)
+        thumbs = re.findall(r'\bthumb:"([^"]*)"', nuxt_raw)
+        video_srcs = re.findall(r'\bvideo_src:"([^"]*)"', nuxt_raw)
+        video_times = re.findall(r'\bvideo_time:"([^"]*)"', nuxt_raw)
+
+        title_refs = re.findall(r'\btitle:([a-zA-Z]\w*)', nuxt_raw)
+        desc_refs = re.findall(r'\bdescription:([a-zA-Z]\w*)', nuxt_raw)
+        # published_at 可能是字符串值或变量引用，统一按顺序提取
+        pub_matches = re.findall(r'\bpublished_at:(?:"([^"]*)"|([a-zA-Z]\w*))', nuxt_raw)
+
+        count = min(len(ids), len(shares), len(thumbs))
+        if count == 0:
+            return []
+
+        def resolve(refs: list, idx: int) -> str:
+            if idx >= len(refs):
+                return ""
+            val = var_map.get(refs[idx])
+            return str(val) if val and val is not None else ""
+
+        def resolve_pub(idx: int) -> str:
+            if idx >= len(pub_matches):
+                return ""
+            str_val, ref_val = pub_matches[idx]
+            if str_val:
+                return str_val
+            if ref_val:
+                val = var_map.get(ref_val)
+                return str(val) if val and val is not None else ""
+            return ""
+
+        now_ts = int(datetime.now().timestamp())
+        videos = []
+        for i in range(count):
+            try:
+                article_id = int(ids[i])
+                share_url = _json.loads(f'"{shares[i]}"')
+                thumb_url = _json.loads(f'"{thumbs[i]}"')
+                video_src = _json.loads(f'"{video_srcs[i]}"') if i < len(video_srcs) else ""
+                video_time = video_times[i] if i < len(video_times) else ""
+            except Exception:
+                continue
+
+            videos.append({
+                "id": article_id,
+                "title": resolve(title_refs, i),
+                "description": resolve(desc_refs, i),
+                "thumb": thumb_url,
+                "share": share_url,
+                "is_video": True,
+                "video_src": video_src,
+                "video_time": video_time,
+                "category": tab_name,
+                "published_at": resolve_pub(i),
+                "sort_timestamp": now_ts,
+            })
+
+        return videos
+
+    # ── 文章详情 ──
+
+    def parse_article_detail(self, response: Dict[str, Any]) -> Optional[Dict[str, str]]:
+        """从文章详情页 HTML 的 __NUXT__ 中提取正文及元信息"""
+        import json as _json
+        html = response.get("html", "")
+        if not html:
+            return None
+
+        nuxt_m = re.search(r'window\.__NUXT__\s*=\s*(.+?);\s*</script>', html, re.DOTALL)
+        if not nuxt_m:
+            return None
+
+        nuxt_raw = nuxt_m.group(1)
+        body_m = re.search(r'\bbody:"((?:[^"\\]|\\.)*)"', nuxt_raw)
+        if not body_m:
+            return None
+
+        try:
+            content = _json.loads(f'"{body_m.group(1)}"')
+        except Exception:
+            return None
+
+        if not content or len(content) < 10:
+            return None
+
+        result = {"content": content}
+
+        for field in ("title", "description", "published_at"):
+            m = re.search(rf'\b{field}:"((?:[^"\\]|\\.)*)"', nuxt_raw)
+            if m:
+                try:
+                    result[field] = _json.loads(f'"{m.group(1)}"')
+                except Exception:
+                    pass
+
+        author_m = re.search(r'\bauthor_name:"((?:[^"\\]|\\.)*)"', nuxt_raw)
+        if author_m:
+            try:
+                result["author"] = _json.loads(f'"{author_m.group(1)}"')
+            except Exception:
+                pass
+
+        return result
+
+    def parse_article_html(self, html: str) -> Optional[Dict[str, str]]:
+        """从 m.dongqiudi.com 文章 HTML 中提取 <article> 标签内的富文本内容"""
+        if not html:
+            return None
+
+        article_m = re.search(r'<article[^>]*>(.*?)</article>', html, re.DOTALL)
+        if not article_m:
+            return None
+
+        article_html = article_m.group(1)
+
+        title = ""
+        h1_m = re.search(r'<h1[^>]*>(.*?)</h1>', article_html, re.DOTALL)
+        if h1_m:
+            title = re.sub(r'<[^>]+>', '', h1_m.group(1)).strip()
+
+        author = ""
+        writer_m = re.search(r'<span[^>]*class="writer"[^>]*>(.*?)</span>', article_html, re.DOTALL)
+        if writer_m:
+            author = re.sub(r'<[^>]+>', '', writer_m.group(1)).strip()
+
+        con_m = re.search(r'<div[^>]*class="con"[^>]*>(.*)', article_html, re.DOTALL)
+        if con_m:
+            content = con_m.group(1).strip()
+            content = re.sub(r'</div>\s*$', '', content, count=1).strip()
+        else:
+            content = ""
+
+        content = re.sub(r'data-src="([^"]*)"', r'src="\1"', content)
+        # 将 GIF 缩略图替换为原始 GIF（data-gif-src 属性中存储了可播放的原图）
+        content = re.sub(
+            r'<img([^>]*)\bsrc="[^"]*"([^>]*)\bdata-gif-src="([^"]*)"',
+            r'<img\1src="\3"\2data-gif-src="\3"',
+            content
+        )
+
+        if not content or len(content) < 20:
+            parts = []
+            if title:
+                parts.append(f'<h1>{title}</h1>')
+            time_text = ""
+            time_m = re.search(r'<time[^>]*>(.*?)</time>', article_html, re.DOTALL)
+            if time_m:
+                time_text = re.sub(r'<[^>]+>', '', time_m.group(1)).strip()
+            if author or time_text:
+                meta = []
+                if author:
+                    meta.append(author)
+                if time_text:
+                    meta.append(time_text)
+                parts.append(f'<p>{" · ".join(meta)}</p>')
+            desc_m = re.search(r'<meta[^>]*name="description"[^>]*content="([^"]*)"', html)
+            if desc_m:
+                desc_text = desc_m.group(1).strip()
+                desc_text = re.sub(r'&lt;[^&]*&gt;', '', desc_text)
+                if desc_text:
+                    parts.append(f'<p>{desc_text}</p>')
+            if not parts:
+                return None
+            content = "\n".join(parts)
+
+        result = {"content": content}
+        if title:
+            result["title"] = title
+        if author:
+            result["author"] = author
+
+        video_urls = re.findall(r'<video[^>]*\bsrc="([^"]+)"', article_html)
+        if not video_urls:
+            video_urls = re.findall(r'<source[^>]*\bsrc="([^"]+)"', article_html)
+        if video_urls:
+            result["video_url"] = video_urls[0]
+
+        return result
+
+    # ── IIFE __NUXT__ 解析 ──
+
+    @staticmethod
+    def _parse_iife_var_map(nuxt_raw: str) -> Dict[str, Any]:
+        """解析 (function(a,b,...){...}(val_a,val_b,...)) 压缩格式，返回变量名->值映射"""
+        import json as _json
+        param_m = re.match(r'^\(function\(([^)]*)\)', nuxt_raw)
+        if not param_m:
+            return {}
+        param_names = [p.strip() for p in param_m.group(1).split(',')]
+
+        last_pos = nuxt_raw.rfind('})(')
+        if last_pos >= 0:
+            args_raw = nuxt_raw[last_pos + 3:-1]
+        else:
+            last_pos = nuxt_raw.rfind('}(')
+            if last_pos >= 0:
+                args_raw = nuxt_raw[last_pos + 2:-2]
+            else:
+                return {}
+
+        args: list = []
+        i = 0
+        n = len(args_raw)
+        while i < n:
+            c = args_raw[i]
+            if c in ' \n\r\t,':
+                i += 1
+                continue
+            if c == '"':
+                j = i + 1
+                while j < n:
+                    if args_raw[j] == '\\':
+                        j += 2
+                    elif args_raw[j] == '"':
+                        break
+                    else:
+                        j += 1
+                try:
+                    args.append(_json.loads(args_raw[i:j + 1]))
+                except Exception:
+                    args.append(args_raw[i + 1:j])
+                i = j + 1
+            elif c in ('{', '['):
+                depth, j, close_c = 0, i, '}' if c == '{' else ']'
+                in_str = False
+                while j < n:
+                    ch = args_raw[j]
+                    if in_str:
+                        if ch == '\\':
+                            j += 1
+                        elif ch == '"':
+                            in_str = False
+                    elif ch == '"':
+                        in_str = True
+                    elif ch == c:
+                        depth += 1
+                    elif ch == close_c:
+                        depth -= 1
+                        if depth == 0:
+                            break
+                    j += 1
+                args.append(args_raw[i:j + 1])
+                i = j + 1
+            else:
+                j = i
+                while j < n and args_raw[j] not in ',)\n':
+                    j += 1
+                token = args_raw[i:j].strip()
+                if token == 'true':
+                    args.append(True)
+                elif token == 'false':
+                    args.append(False)
+                elif token in ('null', 'void 0'):
+                    args.append(None)
+                else:
+                    try:
+                        args.append(int(token))
+                    except ValueError:
+                        try:
+                            args.append(float(token))
+                        except ValueError:
+                            args.append(token)
+                i = j
+
+        return {name: args[idx] for idx, name in enumerate(param_names) if idx < len(args)}
+
+    # ── 工具方法 ──
+
+    def _parse_deduction(self, instruction: Dict) -> Tuple[int, str]:
+        if not instruction:
+            return 0, ""
+        desc = instruction.get("description", "")
+        if not desc:
+            return 0, ""
+        for pattern in [r"扣(\d+)分", r"扣除(\d+)分", r"罚(\d+)分"]:
+            match = re.search(pattern, desc)
+            if match:
+                return int(match.group(1)), desc
+        return 0, desc
+
+    def _parse_recent_form(self, record: str) -> str:
+        if not record:
+            return ""
+        mapping = {"胜": "W", "赢": "W", "平": "D", "负": "L", "输": "L"}
+        return "".join(mapping.get(c, "?") for c in record)[-5:]
+
+    def _normalize_status(self, status: str) -> str:
+        if not status:
+            return "fixture"
+        s = status.lower()
+        if "finished" in s or "完场" in s:
+            return "finished"
+        elif "live" in s or "进行" in s:
+            return "live"
+        elif "postponed" in s or "延期" in s:
+            return "postponed"
+        elif "cancelled" in s or "取消" in s:
+            return "cancelled"
+        return "fixture"
+
+    def _parse_datetime(self, dt_str: Optional[str]) -> Optional[datetime]:
+        if not dt_str:
+            return None
+        for fmt in [
+            "%Y-%m-%d %H:%M:%S", "%Y/%m/%d %H:%M:%S",
+            "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f",
+            "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d %H:%M", "%Y-%m-%d",
+        ]:
+            try:
+                return datetime.strptime(dt_str, fmt)
+            except ValueError:
+                continue
+        return None
+
+    def validate_standing(self, data: Dict) -> List[str]:
+        errors = []
+        for field in ["season_id", "team_id", "rank", "points"]:
+            if field not in data:
+                errors.append(f"缺少字段: {field}")
+        if all(k in data for k in ("played", "won", "drawn", "lost")):
+            if data["played"] != data["won"] + data["drawn"] + data["lost"]:
+                errors.append(f"场次不匹配: {data['played']} != {data['won']}+{data['drawn']}+{data['lost']}")
+        return errors
+
+    def validate_match(self, data: Dict) -> List[str]:
+        errors = []
+        for field in ["match_id", "season_id", "home_team_id", "away_team_id"]:
+            if field not in data:
+                errors.append(f"缺少字段: {field}")
+        if data.get("status") == "finished":
+            if data.get("home_score") is None or data.get("away_score") is None:
+                errors.append("已完场比赛缺少比分")
+        return errors

docker/crawler/src/parser/nuxt_parser.py

@@ -0,0 +1,179 @@
+"""
+从懂球帝 liveDetail 页面的 window.__NUXT__ IIFE 中解析比赛详情数据
+"""
+import re
+from typing import Optional, Dict, Any
+
+
+def parse_js_args(args_str: str) -> list:
+    """解析 JS IIFE 的实参列表，返回 Python 值列表"""
+    results = []
+    i = 0
+    n = len(args_str)
+    while i < n:
+        c = args_str[i]
+        if c in (' ', '\n', '\r', '\t'):
+            i += 1
+            continue
+        if c == ',':
+            i += 1
+            continue
+        if c == '"':
+            j = i + 1
+            while j < n:
+                if args_str[j] == '\\':
+                    j += 2
+                    continue
+                if args_str[j] == '"':
+                    break
+                j += 1
+            raw = args_str[i:j+1]
+            inner = raw[1:-1]
+            if '\\u' in inner:
+                val = inner.encode('utf-8').decode('unicode_escape')
+            else:
+                val = inner.replace('\\n', '\n').replace('\\t', '\t').replace('\\"', '"').replace('\\\\', '\\')
+            results.append(val)
+            i = j + 1
+            continue
+        if args_str[i:i+4] == 'true':
+            results.append(True)
+            i += 4
+            continue
+        if args_str[i:i+5] == 'false':
+            results.append(False)
+            i += 5
+            continue
+        if args_str[i:i+4] == 'null':
+            results.append(None)
+            i += 4
+            continue
+        if args_str[i:i+6] == 'void 0':
+            results.append(None)
+            i += 6
+            continue
+        if c in '0123456789.-':
+            j = i + 1
+            while j < n and args_str[j] in '0123456789.eE+-':
+                j += 1
+            num_str = args_str[i:j]
+            try:
+                val = int(num_str)
+            except ValueError:
+                val = float(num_str)
+            results.append(val)
+            i = j
+            continue
+        if c == '{':
+            depth = 1
+            j = i + 1
+            while j < n and depth > 0:
+                if args_str[j] == '{':
+                    depth += 1
+                elif args_str[j] == '}':
+                    depth -= 1
+                elif args_str[j] == '"':
+                    j += 1
+                    while j < n and args_str[j] != '"':
+                        if args_str[j] == '\\':
+                            j += 1
+                        j += 1
+                j += 1
+            results.append(args_str[i:j])
+            i = j
+            continue
+        if c == '[':
+            depth = 1
+            j = i + 1
+            while j < n and depth > 0:
+                if args_str[j] == '[':
+                    depth += 1
+                elif args_str[j] == ']':
+                    depth -= 1
+                elif args_str[j] == '"':
+                    j += 1
+                    while j < n and args_str[j] != '"':
+                        if args_str[j] == '\\':
+                            j += 1
+                        j += 1
+                j += 1
+            results.append(args_str[i:j])
+            i = j
+            continue
+        j = i
+        while j < n and args_str[j] not in ',)]}':
+            j += 1
+        token = args_str[i:j].strip()
+        results.append(token)
+        i = j
+
+    return results
+
+
+def _decode_str(val_expr: str) -> str:
+    """解码 JS 字符串字面量（去掉引号）"""
+    inner = val_expr[1:-1]
+    if '\\u' in inner:
+        try:
+            return inner.encode('utf-8').decode('unicode_escape')
+        except Exception:
+            return inner
+    return inner.replace('\\n', '\n').replace('\\t', '\t').replace('\\"', '"').replace('\\\\', '\\')
+
+
+def _resolve_value(val_expr: str, var_map: dict):
+    """将 JS 赋值右侧表达式解析为 Python 值"""
+    if val_expr in var_map:
+        return var_map[val_expr]
+    if val_expr.startswith('"') and val_expr.endswith('"'):
+        return _decode_str(val_expr)
+    if val_expr == 'true':
+        return True
+    if val_expr == 'false':
+        return False
+    if val_expr == 'null':
+        return None
+    try:
+        return int(val_expr)
+    except ValueError:
+        pass
+    try:
+        return float(val_expr)
+    except ValueError:
+        pass
+    return val_expr
+
+
+def parse_nuxt_match_sample(html: str) -> Optional[Dict[str, Any]]:
+    """从 liveDetail HTML 中解析 matchSample 对象"""
+    pattern = r'window\.__NUXT__\s*=\s*\(function\(([^)]+)\)\s*\{(.+)\}\((.+)\)\);'
+    m = re.search(pattern, html, re.DOTALL)
+    if not m:
+        return None
+
+    params = [p.strip() for p in m.group(1).split(',')]
+    body = m.group(2)
+    args_str = m.group(3)
+
+    args = parse_js_args(args_str)
+
+    var_map = {}
+    for i, p in enumerate(params):
+        if i < len(args):
+            var_map[p] = args[i]
+
+    ms_var_match = re.search(r'(\w+)\.match_id\s*=\s*(\w+)', body)
+    if not ms_var_match:
+        return None
+
+    ms_var = ms_var_match.group(1)
+
+    assign_pattern = re.compile(rf'\b{re.escape(ms_var)}\.(\w+)\s*=\s*(.+?)\s*;', re.MULTILINE)
+    match_sample = {}
+
+    for am in assign_pattern.finditer(body):
+        field = am.group(1)
+        val_expr = am.group(2).strip()
+        match_sample[field] = _resolve_value(val_expr, var_map)
+
+    return match_sample if match_sample else None

docker/crawler/src/scheduler/cron_scheduler.py

@@ -0,0 +1,376 @@
+"""
+定时调度器
+- 基于 APScheduler 的定时任务管理
+- 支持 Cron 表达式
+"""
+import asyncio
+from typing import Optional
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.scheduler.task_runner import TaskRunner
+
+
+class CronScheduler:
+    """定时调度器"""
+
+    def __init__(self):
+        self._cfg = get_config().scheduler
+        self._runner: Optional[TaskRunner] = None
+        self._scheduler = None
+
+    async def start(self):
+        from apscheduler.schedulers.asyncio import AsyncIOScheduler
+        from apscheduler.triggers.cron import CronTrigger
+
+        self._runner = TaskRunner()
+        await self._runner.initialize()
+
+        self._scheduler = AsyncIOScheduler()
+
+        # 积分榜采集
+        self._scheduler.add_job(
+            self._job_standings,
+            CronTrigger.from_crontab(self._cfg.standings_cron),
+            id="standings",
+            name="积分榜采集",
+        )
+
+        # 赛程采集
+        self._scheduler.add_job(
+            self._job_schedule,
+            CronTrigger.from_crontab(self._cfg.schedule_cron),
+            id="schedule",
+            name="赛程采集",
+        )
+
+        # 新闻资讯采集
+        self._scheduler.add_job(
+            self._job_news,
+            CronTrigger.from_crontab(self._cfg.news_cron),
+            id="news",
+            name="新闻资讯采集",
+        )
+
+        # 实时比赛采集
+        self._scheduler.add_job(
+            self._job_live,
+            CronTrigger.from_crontab(self._cfg.live_cron),
+            id="live",
+            name="实时比赛采集",
+        )
+
+        # 文章详情补全
+        self._scheduler.add_job(
+            self._job_content,
+            CronTrigger.from_crontab(self._cfg.content_cron),
+            id="content",
+            name="文章详情补全",
+        )
+
+        # 视频列表采集
+        self._scheduler.add_job(
+            self._job_video,
+            CronTrigger.from_crontab(self._cfg.video_cron),
+            id="video",
+            name="视频列表采集",
+        )
+
+        # 中超赛程页面采集
+        self._scheduler.add_job(
+            self._job_csl_match,
+            CronTrigger.from_crontab(self._cfg.csl_match_cron),
+            id="csl_match",
+            name="中超赛程页面采集",
+        )
+
+        # NBA赛程页面采集
+        self._scheduler.add_job(
+            self._job_nba_match,
+            CronTrigger.from_crontab(self._cfg.nba_match_cron),
+            id="nba_match",
+            name="NBA赛程页面采集",
+        )
+
+        # CBA赛程页面采集
+        self._scheduler.add_job(
+            self._job_cba_match,
+            CronTrigger.from_crontab(self._cfg.cba_match_cron),
+            id="cba_match",
+            name="CBA赛程页面采集",
+        )
+
+        # 英超赛程页面采集
+        self._scheduler.add_job(
+            self._job_epl_match,
+            CronTrigger.from_crontab(self._cfg.epl_match_cron),
+            id="epl_match",
+            name="英超赛程页面采集",
+        )
+
+        # 德甲赛程页面采集
+        self._scheduler.add_job(
+            self._job_bundesliga_match,
+            CronTrigger.from_crontab(self._cfg.bundesliga_match_cron),
+            id="bundesliga_match",
+            name="德甲赛程页面采集",
+        )
+
+        # 西甲赛程页面采集
+        self._scheduler.add_job(
+            self._job_laliga_match,
+            CronTrigger.from_crontab(self._cfg.laliga_match_cron),
+            id="laliga_match",
+            name="西甲赛程页面采集",
+        )
+
+        # 意甲赛程页面采集
+        self._scheduler.add_job(
+            self._job_seriea_match,
+            CronTrigger.from_crontab(self._cfg.seriea_match_cron),
+            id="seriea_match",
+            name="意甲赛程页面采集",
+        )
+
+        # 法甲赛程页面采集
+        self._scheduler.add_job(
+            self._job_ligue1_match,
+            CronTrigger.from_crontab(self._cfg.ligue1_match_cron),
+            id="ligue1_match",
+            name="法甲赛程页面采集",
+        )
+
+        # 欧冠赛程页面采集
+        self._scheduler.add_job(
+            self._job_ucl_match,
+            CronTrigger.from_crontab(self._cfg.ucl_match_cron),
+            id="ucl_match",
+            name="欧冠赛程页面采集",
+        )
+
+        # 欧联赛程页面采集
+        self._scheduler.add_job(
+            self._job_uel_match,
+            CronTrigger.from_crontab(self._cfg.uel_match_cron),
+            id="uel_match",
+            name="欧联赛程页面采集",
+        )
+
+        # 网球赛程页面采集
+        self._scheduler.add_job(
+            self._job_tennis_match,
+            CronTrigger.from_crontab(self._cfg.tennis_match_cron),
+            id="tennis_match",
+            name="网球赛程页面采集",
+        )
+
+        # 电竞赛程页面采集
+        self._scheduler.add_job(
+            self._job_esports_match,
+            CronTrigger.from_crontab(self._cfg.esports_match_cron),
+            id="esports_match",
+            name="电竞赛程页面采集",
+        )
+
+        # 体坛赛程页面采集
+        self._scheduler.add_job(
+            self._job_sports_match,
+            CronTrigger.from_crontab(self._cfg.sports_match_cron),
+            id="sports_match",
+            name="体坛赛程页面采集",
+        )
+
+        # 联赛资讯采集
+        self._scheduler.add_job(
+            self._job_league_news,
+            CronTrigger.from_crontab(self._cfg.league_news_cron),
+            id="league_news",
+            name="联赛资讯采集",
+        )
+
+        # 文章内容补全
+        self._scheduler.add_job(
+            self._job_article_content,
+            CronTrigger.from_crontab(self._cfg.article_content_cron),
+            id="article_content",
+            name="文章内容补全",
+        )
+
+        self._scheduler.start()
+        logger.info("定时调度器已启动")
+        logger.info(f"  积分榜: {self._cfg.standings_cron}")
+        logger.info(f"  赛程:   {self._cfg.schedule_cron}")
+        logger.info(f"  新闻:   {self._cfg.news_cron}")
+        logger.info(f"  联赛资讯: {self._cfg.league_news_cron}")
+        logger.info(f"  文章补全: {self._cfg.article_content_cron}")
+        logger.info(f"  实时:   {self._cfg.live_cron}")
+        logger.info(f"  详情:   {self._cfg.content_cron}")
+        logger.info(f"  视频:   {self._cfg.video_cron}")
+        logger.info(f"  中超:   {self._cfg.csl_match_cron}")
+        logger.info(f"  NBA:   {self._cfg.nba_match_cron}")
+        logger.info(f"  CBA:   {self._cfg.cba_match_cron}")
+        logger.info(f"  英超:   {self._cfg.epl_match_cron}")
+        logger.info(f"  德甲:   {self._cfg.bundesliga_match_cron}")
+        logger.info(f"  西甲:   {self._cfg.laliga_match_cron}")
+        logger.info(f"  意甲:   {self._cfg.seriea_match_cron}")
+        logger.info(f"  法甲:   {self._cfg.ligue1_match_cron}")
+        logger.info(f"  欧冠:   {self._cfg.ucl_match_cron}")
+        logger.info(f"  欧联:   {self._cfg.uel_match_cron}")
+        logger.info(f"  网球:   {self._cfg.tennis_match_cron}")
+        logger.info(f"  电竞:   {self._cfg.esports_match_cron}")
+        logger.info(f"  体坛:   {self._cfg.sports_match_cron}")
+
+    async def stop(self):
+        if self._scheduler:
+            self._scheduler.shutdown(wait=False)
+        if self._runner:
+            await self._runner.close()
+        logger.info("定时调度器已停止")
+
+    async def _job_standings(self):
+        logger.info("[CRON] 触发积分榜采集")
+        try:
+            await self._runner.crawl_all_leagues("standings")
+        except Exception as e:
+            logger.error(f"[CRON] 积分榜采集失败: {e}")
+
+    async def _job_schedule(self):
+        logger.info("[CRON] 触发赛程采集")
+        try:
+            await self._runner.crawl_all_leagues("schedule")
+        except Exception as e:
+            logger.error(f"[CRON] 赛程采集失败: {e}")
+
+    async def _job_news(self):
+        logger.info("[CRON] 触发新闻资讯采集")
+        try:
+            await self._runner.crawl_news()
+        except Exception as e:
+            logger.error(f"[CRON] 新闻采集失败: {e}")
+
+    async def _job_live(self):
+        logger.info("[CRON] 触发实时比赛采集")
+        try:
+            await self._runner.crawl_live()
+        except Exception as e:
+            logger.error(f"[CRON] 实时比赛采集失败: {e}")
+
+    async def _job_content(self):
+        logger.info("[CRON] 触发文章详情补全")
+        try:
+            await self._runner.crawl_article_details(batch_size=3)
+        except Exception as e:
+            logger.error(f"[CRON] 文章详情补全失败: {e}")
+
+    async def _job_video(self):
+        logger.info("[CRON] 触发视频列表采集")
+        try:
+            await self._runner.crawl_videos()
+        except Exception as e:
+            logger.error(f"[CRON] 视频列表采集失败: {e}")
+
+    async def _job_csl_match(self):
+        logger.info("[CRON] 触发中超赛程页面采集")
+        try:
+            await self._runner.crawl_csl_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 中超赛程采集失败: {e}")
+
+    async def _job_nba_match(self):
+        logger.info("[CRON] 触发NBA赛程页面采集")
+        try:
+            await self._runner.crawl_nba_matches()
+        except Exception as e:
+            logger.error(f"[CRON] NBA赛程采集失败: {e}")
+
+    async def _job_cba_match(self):
+        logger.info("[CRON] 触发CBA赛程页面采集")
+        try:
+            await self._runner.crawl_cba_matches()
+        except Exception as e:
+            logger.error(f"[CRON] CBA赛程采集失败: {e}")
+
+    async def _job_epl_match(self):
+        logger.info("[CRON] 触发英超赛程页面采集")
+        try:
+            await self._runner.crawl_epl_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 英超赛程采集失败: {e}")
+
+    async def _job_bundesliga_match(self):
+        logger.info("[CRON] 触发德甲赛程页面采集")
+        try:
+            await self._runner.crawl_bundesliga_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 德甲赛程采集失败: {e}")
+
+    async def _job_laliga_match(self):
+        logger.info("[CRON] 触发西甲赛程页面采集")
+        try:
+            await self._runner.crawl_laliga_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 西甲赛程采集失败: {e}")
+
+    async def _job_seriea_match(self):
+        logger.info("[CRON] 触发意甲赛程页面采集")
+        try:
+            await self._runner.crawl_seriea_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 意甲赛程采集失败: {e}")
+
+    async def _job_ligue1_match(self):
+        logger.info("[CRON] 触发法甲赛程页面采集")
+        try:
+            await self._runner.crawl_ligue1_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 法甲赛程采集失败: {e}")
+
+    async def _job_ucl_match(self):
+        logger.info("[CRON] 触发欧冠赛程页面采集")
+        try:
+            await self._runner.crawl_ucl_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 欧冠赛程采集失败: {e}")
+
+    async def _job_uel_match(self):
+        logger.info("[CRON] 触发欧联赛程页面采集")
+        try:
+            await self._runner.crawl_uel_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 欧联赛程采集失败: {e}")
+
+    async def _job_tennis_match(self):
+        logger.info("[CRON] 触发网球赛程页面采集")
+        try:
+            await self._runner.crawl_tennis_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 网球赛程采集失败: {e}")
+
+    async def _job_esports_match(self):
+        logger.info("[CRON] 触发电竞赛程页面采集")
+        try:
+            await self._runner.crawl_esports_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 电竞赛程采集失败: {e}")
+
+    async def _job_sports_match(self):
+        logger.info("[CRON] 触发体坛赛程页面采集")
+        try:
+            await self._runner.crawl_sports_matches()
+        except Exception as e:
+            logger.error(f"[CRON] 体坛赛程采集失败: {e}")
+
+    async def _job_league_news(self):
+        logger.info("[CRON] 触发联赛资讯采集")
+        try:
+            await self._runner.crawl_league_news()
+        except Exception as e:
+            logger.error(f"[CRON] 联赛资讯采集失败: {e}")
+
+    async def _job_article_content(self):
+        logger.info("[CRON] 触发文章内容补全")
+        try:
+            await self._runner.crawl_article_content()
+        except Exception as e:
+            logger.error(f"[CRON] 文章内容补全失败: {e}")