迁移目录

docker/crawler/src/anti_detect/fingerprint.py

@@ -0,0 +1,237 @@
+"""
+浏览器指纹生成与管理
+- 生成逼真的浏览器指纹（UA、屏幕、Canvas、WebGL、TLS）
+- 指纹池轮换、冷却、黑名单机制
+"""
+import asyncio
+import hashlib
+import random
+import time
+from datetime import datetime, timedelta
+from typing import Dict, Any, List, Optional, Set
+
+from loguru import logger
+from fake_useragent import UserAgent
+
+from src.core.config import get_config
+
+
+# ── 常量 ──────────────────────────────────────────────
+
+CHROME_VERSIONS = [
+    "120.0.6099.109", "121.0.6167.85", "122.0.6261.57",
+    "123.0.6312.86", "124.0.6367.91", "125.0.6422.60",
+]
+
+SCREEN_RESOLUTIONS = [
+    {"width": 1920, "height": 1080, "ratio": 1.0},
+    {"width": 1366, "height": 768, "ratio": 1.0},
+    {"width": 1536, "height": 864, "ratio": 1.25},
+    {"width": 2560, "height": 1440, "ratio": 1.5},
+    {"width": 1440, "height": 900, "ratio": 1.0},
+    {"width": 1680, "height": 1050, "ratio": 1.0},
+    {"width": 1280, "height": 800, "ratio": 1.0},
+    {"width": 3840, "height": 2160, "ratio": 2.0},
+]
+
+WEBGL_RENDERERS = [
+    "ANGLE (Intel, Intel(R) UHD Graphics 630 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (Intel, Intel(R) UHD Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (AMD, AMD Radeon(TM) Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (NVIDIA, NVIDIA GeForce GTX 1060 6GB Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)",
+    "ANGLE (Intel, Intel(R) Iris(R) Xe Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
+]
+
+FONTS_POOL = [
+    "Arial", "Arial Black", "Calibri", "Cambria", "Consolas",
+    "Courier New", "Georgia", "Impact", "Segoe UI", "Tahoma",
+    "Times New Roman", "Trebuchet MS", "Verdana",
+    "Microsoft YaHei", "SimHei", "SimSun", "NSimSun", "FangSong", "KaiTi",
+]
+
+SEC_CH_UA_TEMPLATES = {
+    "chrome": '"Chromium";v="{major}", "Google Chrome";v="{major}", "Not-A.Brand";v="99"',
+    "edge": '"Chromium";v="{major}", "Microsoft Edge";v="{major}", "Not-A.Brand";v="99"',
+}
+
+
+class FingerprintGenerator:
+    """指纹生成器"""
+
+    def __init__(self):
+        try:
+            self._ua = UserAgent(browsers=["chrome", "edge"], os=["windows", "macos"])
+        except Exception:
+            self._ua = None
+
+    def generate(self) -> Dict[str, Any]:
+        browser = random.choice(["chrome", "edge"])
+        version = random.choice(CHROME_VERSIONS)
+        major = version.split(".")[0]
+        ua_string = self._build_ua(browser, version)
+        screen = random.choice(SCREEN_RESOLUTIONS)
+        fp_id = f"fp_{int(time.time())}_{random.randint(1000, 9999)}"
+
+        return {
+            "id": fp_id,
+            "browser": browser,
+            "version": version,
+            "user_agent": ua_string,
+            "screen": screen,
+            "sec_ch_ua": SEC_CH_UA_TEMPLATES.get(browser, SEC_CH_UA_TEMPLATES["chrome"]).format(major=major),
+            "http_headers": self._build_headers(ua_string, browser, major),
+            "canvas_hash": hashlib.md5(f"canvas_{random.randint(100000, 999999)}".encode()).hexdigest()[:16],
+            "webgl": {
+                "vendor": "Google Inc. (Intel)",
+                "renderer": random.choice(WEBGL_RENDERERS),
+            },
+            "fonts": sorted(random.sample(FONTS_POOL, k=random.randint(10, len(FONTS_POOL)))),
+            "timezone": "Asia/Shanghai",
+            "language": "zh-CN",
+            "created_at": datetime.now().isoformat(),
+            "success_rate": 1.0,
+            "usage_count": 0,
+            "is_active": True,
+        }
+
+    def _build_ua(self, browser: str, version: str) -> str:
+        os_strings = [
+            "Windows NT 10.0; Win64; x64",
+            "Windows NT 10.0; Win64; x64",
+            "Macintosh; Intel Mac OS X 10_15_7",
+        ]
+        os_str = random.choice(os_strings)
+        major = version.split(".")[0]
+
+        if browser == "edge":
+            return (
+                f"Mozilla/5.0 ({os_str}) AppleWebKit/537.36 "
+                f"(KHTML, like Gecko) Chrome/{version} Safari/537.36 Edg/{version}"
+            )
+        return (
+            f"Mozilla/5.0 ({os_str}) AppleWebKit/537.36 "
+            f"(KHTML, like Gecko) Chrome/{version} Safari/537.36"
+        )
+
+    def _build_headers(self, ua: str, browser: str, major: str) -> Dict[str, str]:
+        headers = {
+            "User-Agent": ua,
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Referer": "https://www.dongqiudi.com/",
+            "Origin": "https://www.dongqiudi.com",
+            "Sec-Fetch-Dest": "empty",
+            "Sec-Fetch-Mode": "cors",
+            "Sec-Fetch-Site": "same-site",
+            "Sec-Ch-Ua": SEC_CH_UA_TEMPLATES.get(browser, SEC_CH_UA_TEMPLATES["chrome"]).format(major=major),
+            "Sec-Ch-Ua-Mobile": "?0",
+            "Sec-Ch-Ua-Platform": '"Windows"',
+        }
+        return headers
+
+
+class FingerprintPool:
+    """指纹池管理器"""
+
+    def __init__(self):
+        cfg = get_config().anti_detect.fingerprint
+        self._pool_size = cfg.pool_size
+        self._min_success_rate = cfg.min_success_rate
+        self._cooldown = cfg.cooldown
+        self._generator = FingerprintGenerator()
+        self._pool: Dict[str, Dict[str, Any]] = {}
+        self._blacklist: Set[str] = set()
+        self._lock = asyncio.Lock()
+
+    async def initialize(self, size: Optional[int] = None):
+        size = size or self._pool_size
+        async with self._lock:
+            for _ in range(size):
+                fp = self._generator.generate()
+                self._pool[fp["id"]] = fp
+        logger.info(f"指纹池初始化完成，共 {len(self._pool)} 个指纹")
+
+    async def acquire(self) -> Dict[str, Any]:
+        async with self._lock:
+            now = datetime.now()
+            candidates = []
+
+            for fp_id, fp in self._pool.items():
+                if fp_id in self._blacklist or not fp.get("is_active", True):
+                    continue
+                if fp.get("success_rate", 1.0) < self._min_success_rate:
+                    fp["is_active"] = False
+                    continue
+                last_used = fp.get("last_used")
+                if last_used:
+                    last_dt = datetime.fromisoformat(last_used) if isinstance(last_used, str) else last_used
+                    if (now - last_dt).total_seconds() < self._cooldown:
+                        continue
+                candidates.append(fp)
+
+            if not candidates:
+                logger.warning("指纹池耗尽，生成新指纹")
+                fp = self._generator.generate()
+                self._pool[fp["id"]] = fp
+                candidates = [fp]
+
+            candidates.sort(key=lambda x: (x.get("usage_count", 0), -x.get("success_rate", 1.0)))
+            chosen = candidates[0]
+            chosen["last_used"] = now.isoformat()
+            chosen["usage_count"] = chosen.get("usage_count", 0) + 1
+            return chosen.copy()
+
+    async def mark_success(self, fp_id: str):
+        async with self._lock:
+            fp = self._pool.get(fp_id)
+            if not fp:
+                return
+            total = fp.get("usage_count", 1)
+            fails = fp.get("fail_count", 0)
+            fp["success_rate"] = 1.0 - (fails / total) if total > 0 else 1.0
+
+    async def mark_failure(self, fp_id: str, error_type: str = ""):
+        async with self._lock:
+            fp = self._pool.get(fp_id)
+            if not fp:
+                return
+            fp["fail_count"] = fp.get("fail_count", 0) + 1
+            total = fp.get("usage_count", 1)
+            fails = fp["fail_count"]
+            fp["success_rate"] = 1.0 - (fails / total) if total > 0 else 0.0
+
+            if error_type in ("ip_blocked", "captcha", "fingerprint_detected"):
+                self._blacklist.add(fp_id)
+                fp["is_active"] = False
+                logger.warning(f"指纹 {fp_id} 被加入黑名单: {error_type}")
+
+    async def cleanup(self):
+        async with self._lock:
+            to_remove = [
+                fp_id for fp_id, fp in self._pool.items()
+                if not fp.get("is_active") and fp.get("success_rate", 1.0) < self._min_success_rate * 0.5
+            ]
+            for fp_id in to_remove:
+                self._pool.pop(fp_id, None)
+                self._blacklist.discard(fp_id)
+
+            deficit = self._pool_size - len(self._pool)
+            if deficit > 0:
+                for _ in range(deficit):
+                    fp = self._generator.generate()
+                    self._pool[fp["id"]] = fp
+                logger.info(f"指纹池补充 {deficit} 个新指纹")
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        active = sum(1 for fp in self._pool.values() if fp.get("is_active", True))
+        rates = [fp.get("success_rate", 1.0) for fp in self._pool.values()]
+        return {
+            "total": len(self._pool),
+            "active": active,
+            "blacklisted": len(self._blacklist),
+            "avg_success_rate": sum(rates) / len(rates) if rates else 1.0,
+        }

docker/crawler/src/anti_detect/proxy_manager.py

@@ -0,0 +1,162 @@
+"""
+代理管理器
+- 代理池维护、健康检测、自动轮换
+- 支持 HTTP / SOCKS5 代理
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, List, Optional
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import ProxyError
+
+
+class Proxy:
+    __slots__ = (
+        "url", "protocol", "host", "port", "username", "password",
+        "success_count", "fail_count", "last_used", "last_check",
+        "latency", "is_active",
+    )
+
+    def __init__(self, url: str, protocol: str = "http"):
+        self.url = url
+        self.protocol = protocol
+        parts = url.replace("http://", "").replace("https://", "").replace("socks5://", "")
+        auth_host = parts.split("@")
+        if len(auth_host) == 2:
+            auth = auth_host[0].split(":")
+            self.username = auth[0]
+            self.password = auth[1] if len(auth) > 1 else ""
+            host_port = auth_host[1].split(":")
+        else:
+            self.username = ""
+            self.password = ""
+            host_port = auth_host[0].split(":")
+        self.host = host_port[0]
+        self.port = int(host_port[1]) if len(host_port) > 1 else 80
+
+        self.success_count = 0
+        self.fail_count = 0
+        self.last_used = 0.0
+        self.last_check = 0.0
+        self.latency = 0.0
+        self.is_active = True
+
+    @property
+    def success_rate(self) -> float:
+        total = self.success_count + self.fail_count
+        return self.success_count / total if total > 0 else 1.0
+
+    def mark_success(self, latency: float):
+        self.success_count += 1
+        self.latency = latency
+        self.last_used = time.time()
+
+    def mark_failure(self):
+        self.fail_count += 1
+        self.last_used = time.time()
+        cfg = get_config().anti_detect.proxy
+        if self.fail_count >= cfg.max_failures:
+            self.is_active = False
+            logger.warning(f"代理 {self.host}:{self.port} 因连续失败被停用")
+
+
+class ProxyManager:
+    """代理池管理器"""
+
+    def __init__(self):
+        self._proxies: List[Proxy] = []
+        self._lock = asyncio.Lock()
+        self._enabled = get_config().anti_detect.proxy.enabled
+
+    @property
+    def enabled(self) -> bool:
+        return self._enabled and len(self._proxies) > 0
+
+    async def add_proxy(self, url: str, protocol: str = "http"):
+        async with self._lock:
+            proxy = Proxy(url, protocol)
+            self._proxies.append(proxy)
+            logger.info(f"添加代理: {proxy.host}:{proxy.port}")
+
+    async def add_proxies(self, urls: List[str], protocol: str = "http"):
+        for url in urls:
+            await self.add_proxy(url, protocol)
+
+    async def get_proxy(self) -> Optional[str]:
+        if not self._enabled:
+            return None
+
+        async with self._lock:
+            active = [p for p in self._proxies if p.is_active]
+            if not active:
+                logger.warning("无可用代理")
+                return None
+
+            active.sort(key=lambda p: (p.success_rate, -p.latency), reverse=True)
+
+            top_n = max(1, len(active) // 3)
+            chosen = random.choice(active[:top_n])
+            return chosen.url
+
+    async def mark_success(self, proxy_url: str, latency: float):
+        async with self._lock:
+            for p in self._proxies:
+                if p.url == proxy_url:
+                    p.mark_success(latency)
+                    break
+
+    async def mark_failure(self, proxy_url: str):
+        async with self._lock:
+            for p in self._proxies:
+                if p.url == proxy_url:
+                    p.mark_failure()
+                    break
+
+    async def health_check(self):
+        """批量检测代理健康状态"""
+        import aiohttp
+
+        test_url = get_config().anti_detect.proxy.test_url
+        tasks = []
+
+        async def _check(proxy: Proxy):
+            try:
+                start = time.time()
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(
+                        test_url,
+                        proxy=proxy.url,
+                        timeout=aiohttp.ClientTimeout(total=10),
+                    ) as resp:
+                        if resp.status == 200:
+                            proxy.latency = time.time() - start
+                            proxy.is_active = True
+                            proxy.last_check = time.time()
+                        else:
+                            proxy.is_active = False
+            except Exception:
+                proxy.is_active = False
+
+        async with self._lock:
+            for p in self._proxies:
+                tasks.append(_check(p))
+
+        await asyncio.gather(*tasks, return_exceptions=True)
+        active_count = sum(1 for p in self._proxies if p.is_active)
+        logger.info(f"代理健康检查完成: {active_count}/{len(self._proxies)} 可用")
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        active = sum(1 for p in self._proxies if p.is_active)
+        return {
+            "total": len(self._proxies),
+            "active": active,
+            "avg_latency": (
+                sum(p.latency for p in self._proxies if p.is_active) / active
+                if active else 0
+            ),
+        }

docker/crawler/src/anti_detect/stealth_browser.py

@@ -0,0 +1,285 @@
+"""
+Playwright 隐身浏览器
+- 深度反检测：移除 webdriver 痕迹、模拟人类行为
+- 自动处理验证码页面、JS 挑战
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, Optional, List
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import CaptchaError, BlockedError
+
+
+class StealthBrowser:
+    """隐身浏览器 - Playwright + 深度反检测"""
+
+    def __init__(self):
+        cfg = get_config().anti_detect.playwright
+        self._browser_type = cfg.browser
+        self._headless = cfg.headless
+        self._use_stealth = cfg.stealth
+        self._viewport = cfg.viewport
+        self._anti = cfg.anti_features
+        self._playwright = None
+        self._browser = None
+        self._context = None
+        self._running = False
+
+    async def start(self):
+        if self._running:
+            return
+
+        from playwright.async_api import async_playwright
+
+        self._playwright = await async_playwright().start()
+
+        launch_args = [
+            "--disable-blink-features=AutomationControlled",
+            "--disable-dev-shm-usage",
+            "--no-sandbox",
+            "--disable-infobars",
+            "--disable-background-timer-throttling",
+            "--disable-backgrounding-occluded-windows",
+            "--disable-renderer-backgrounding",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+
+        if self._anti.disable_webrtc:
+            launch_args.append("--disable-webrtc")
+
+        browser_launcher = getattr(self._playwright, self._browser_type)
+        self._browser = await browser_launcher.launch(
+            headless=self._headless,
+            args=launch_args,
+        )
+
+        context_opts = {
+            "viewport": {"width": self._viewport.width, "height": self._viewport.height},
+            "locale": "zh-CN",
+            "timezone_id": "Asia/Shanghai",
+        }
+
+        if self._anti.random_viewport:
+            w_offset = random.randint(-50, 50)
+            h_offset = random.randint(-30, 30)
+            context_opts["viewport"] = {
+                "width": self._viewport.width + w_offset,
+                "height": self._viewport.height + h_offset,
+            }
+
+        self._context = await self._browser.new_context(**context_opts)
+        self._running = True
+        logger.info(f"隐身浏览器已启动 ({self._browser_type}, headless={self._headless})")
+
+    async def stop(self):
+        if self._context:
+            await self._context.close()
+            self._context = None
+        if self._browser:
+            await self._browser.close()
+            self._browser = None
+        if self._playwright:
+            await self._playwright.stop()
+            self._playwright = None
+        self._running = False
+        logger.info("隐身浏览器已关闭")
+
+    async def new_page(self):
+        if not self._running:
+            await self.start()
+
+        page = await self._context.new_page()
+
+        if self._use_stealth:
+            await self._apply_stealth(page)
+
+        return page
+
+    async def _apply_stealth(self, page):
+        """应用全套反检测措施"""
+        try:
+            from playwright_stealth import stealth_async
+            await stealth_async(page)
+            logger.debug("playwright-stealth 反检测已应用")
+        except ImportError:
+            logger.debug("playwright-stealth 未安装，使用内置反检测")
+
+        await self._inject_anti_detection(page)
+
+    async def _inject_anti_detection(self, page):
+        """注入反检测 JS"""
+
+        if self._anti.remove_webdriver:
+            await page.add_init_script("""
+                Object.defineProperty(navigator, 'webdriver', {
+                    get: () => undefined,
+                });
+                // 删除 __webdriver_evaluate 等属性
+                delete navigator.__webdriver_evaluate;
+                delete navigator.__webdriver_unwrap;
+            """)
+
+        if self._anti.fake_plugins:
+            await page.add_init_script("""
+                Object.defineProperty(navigator, 'plugins', {
+                    get: () => {
+                        const plugins = [
+                            {name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer'},
+                            {name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai'},
+                            {name: 'Native Client', filename: 'internal-nacl-plugin'},
+                        ];
+                        plugins.length = 3;
+                        return plugins;
+                    },
+                });
+                Object.defineProperty(navigator, 'mimeTypes', {
+                    get: () => {
+                        const mimes = [
+                            {type: 'application/pdf', suffixes: 'pdf'},
+                            {type: 'application/x-nacl', suffixes: ''},
+                        ];
+                        mimes.length = 2;
+                        return mimes;
+                    },
+                });
+            """)
+
+        if self._anti.fake_languages:
+            await page.add_init_script("""
+                Object.defineProperty(navigator, 'languages', {
+                    get: () => ['zh-CN', 'zh', 'en-US', 'en'],
+                });
+                Object.defineProperty(navigator, 'language', {
+                    get: () => 'zh-CN',
+                });
+            """)
+
+        if self._anti.fake_chrome_runtime:
+            await page.add_init_script("""
+                window.chrome = {
+                    app: { isInstalled: false, InstallState: {DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed'}, RunningState: {CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running'} },
+                    runtime: { OnInstalledReason: {CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update'}, OnRestartRequiredReason: {APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic'}, PlatformArch: {ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64'}, PlatformNaclArch: {ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64'}, PlatformOs: {ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win'}, RequestUpdateCheckStatus: {NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available'} },
+                };
+            """)
+
+        # 隐藏自动化 permissions
+        await page.add_init_script("""
+            const originalQuery = window.navigator.permissions.query;
+            window.navigator.permissions.query = (parameters) => (
+                parameters.name === 'notifications' ?
+                    Promise.resolve({ state: Notification.permission }) :
+                    originalQuery(parameters)
+            );
+        """)
+
+        # 伪造 connection.rtt
+        await page.add_init_script("""
+            if (navigator.connection) {
+                Object.defineProperty(navigator.connection, 'rtt', { get: () => 50 });
+            }
+        """)
+
+    async def human_scroll(self, page, times: int = 3):
+        """模拟人类滚动行为"""
+        for _ in range(times):
+            scroll_y = random.randint(100, 400)
+            await page.mouse.wheel(0, scroll_y)
+            await asyncio.sleep(random.uniform(0.3, 1.0))
+
+    async def human_mouse_move(self, page, x: int, y: int):
+        """模拟人类鼠标移动（贝塞尔曲线）"""
+        steps = random.randint(10, 25)
+        current_x, current_y = 0, 0
+        for i in range(steps):
+            t = (i + 1) / steps
+            next_x = int(current_x + (x - current_x) * t + random.randint(-5, 5))
+            next_y = int(current_y + (y - current_y) * t + random.randint(-3, 3))
+            await page.mouse.move(next_x, next_y)
+            await asyncio.sleep(random.uniform(0.01, 0.05))
+
+    async def fetch_page(
+        self,
+        url: str,
+        wait_selector: Optional[str] = None,
+        wait_time: float = 0,
+        extract_json: bool = False,
+    ) -> Dict[str, Any]:
+        """使用隐身浏览器获取页面"""
+        page = None
+        try:
+            page = await self.new_page()
+            start = time.time()
+
+            await page.goto(url, wait_until="networkidle", timeout=30000)
+            nav_time = time.time() - start
+
+            if wait_selector:
+                await page.wait_for_selector(wait_selector, timeout=10000)
+
+            if self._anti.human_mouse:
+                await self.human_mouse_move(page, random.randint(200, 800), random.randint(200, 600))
+
+            if self._anti.random_scroll:
+                await self.human_scroll(page, times=random.randint(1, 3))
+
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+
+            content = await page.content()
+            title = await page.title()
+            total_time = time.time() - start
+
+            # 检测反爬页面
+            if self._is_blocked_page(content, title):
+                raise BlockedError(f"检测到反爬页面: {title}")
+
+            result = {
+                "success": True,
+                "url": page.url,
+                "title": title,
+                "content": content,
+                "content_size": len(content),
+                "nav_time": nav_time,
+                "total_time": total_time,
+            }
+
+            if extract_json:
+                json_data = await self._extract_json(page)
+                if json_data:
+                    result["json_data"] = json_data
+
+            return result
+
+        except (BlockedError, CaptchaError):
+            raise
+        except Exception as e:
+            logger.error(f"页面获取失败 {url}: {e}")
+            return {"success": False, "url": url, "error": str(e)}
+        finally:
+            if page:
+                await page.close()
+
+    def _is_blocked_page(self, content: str, title: str) -> bool:
+        blocked_keywords = ["验证码", "Captcha", "Access Denied", "403 Forbidden", "请稍后", "访问频繁"]
+        text = (content[:3000] + title).lower()
+        return any(kw.lower() in text for kw in blocked_keywords)
+
+    async def _extract_json(self, page) -> Optional[Any]:
+        """从页面中提取 JSON 数据（用于 API 页面）"""
+        try:
+            text = await page.evaluate("() => document.body.innerText")
+            import json
+            return json.loads(text)
+        except Exception:
+            return None
+
+    async def __aenter__(self):
+        await self.start()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.stop()

docker/crawler/src/anti_detect/tls_client.py

@@ -0,0 +1,148 @@
+"""
+TLS 指纹伪装客户端
+使用 curl_cffi 模拟真实浏览器的 TLS 握手指纹 (JA3/JA4)
+这是绕过懂球帝 TLS 指纹检测的核心模块
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, Optional
+from urllib.parse import urlencode
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import NetworkError, RateLimitError, BlockedError, CaptchaError
+
+
+# curl_cffi 支持的浏览器指纹列表
+IMPERSONATE_TARGETS = [
+    "chrome120", "chrome123", "chrome124",
+    "edge101", "edge99",
+    "safari15_5", "safari17_0",
+]
+
+
+class TLSClient:
+    """
+    基于 curl_cffi 的 TLS 指纹伪装 HTTP 客户端
+    能够模拟真实浏览器的 JA3 指纹，绕过 TLS 指纹检测
+    """
+
+    def __init__(self, impersonate: Optional[str] = None):
+        cfg = get_config()
+        self._impersonate = impersonate or cfg.anti_detect.tls.impersonate
+        self._timeout = cfg.crawler.request_timeout
+        self._session = None
+        self._request_count = 0
+        self._max_per_session = cfg.crawler.session.max_requests_per_session
+        self._created_at = None
+
+    async def _ensure_session(self):
+        if self._session is None or self._should_rotate():
+            await self._rotate_session()
+
+    def _should_rotate(self) -> bool:
+        if self._request_count >= self._max_per_session:
+            return True
+        cfg = get_config().crawler.session
+        if self._created_at and (time.time() - self._created_at) > cfg.rotate_interval:
+            return True
+        return False
+
+    async def _rotate_session(self):
+        await self.close()
+        try:
+            from curl_cffi.requests import AsyncSession
+            target = random.choice(IMPERSONATE_TARGETS) if self._impersonate == "random" else self._impersonate
+            self._session = AsyncSession(impersonate=target, timeout=self._timeout)
+            self._request_count = 0
+            self._created_at = time.time()
+            logger.debug(f"TLS 会话已创建，伪装: {target}")
+        except ImportError:
+            logger.warning("curl_cffi 未安装，回退到 aiohttp（无 TLS 指纹伪装）")
+            self._session = None
+            raise
+
+    async def get(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        proxy: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        await self._ensure_session()
+        self._request_count += 1
+
+        full_url = f"{url}?{urlencode(params)}" if params else url
+        logger.info(f"[tls] GET {full_url}")
+
+        start = time.time()
+        try:
+            resp = await self._session.get(
+                url,
+                params=params,
+                headers=headers,
+                proxy=proxy,
+                allow_redirects=True,
+            )
+            elapsed = time.time() - start
+
+            if resp.status_code == 200:
+                try:
+                    data = resp.json()
+                except Exception:
+                    data = resp.text
+                resp_summary = str(data)[:300] if data else '(empty)'
+                logger.info(f"[tls] 200 OK {elapsed:.2f}s {full_url} resp={resp_summary}")
+                return {
+                    "success": True,
+                    "status": resp.status_code,
+                    "data": data,
+                    "elapsed": elapsed,
+                    "headers": dict(resp.headers),
+                }
+
+            elif resp.status_code == 429:
+                logger.warning(f"[tls] 429 RateLimit {elapsed:.2f}s {full_url}")
+                retry_after = int(resp.headers.get("Retry-After", 60))
+                raise RateLimitError(retry_after=retry_after)
+
+            elif resp.status_code == 403:
+                logger.warning(f"[tls] 403 Forbidden {elapsed:.2f}s {full_url}")
+                raise BlockedError(f"403 Forbidden: {url}")
+
+            elif resp.status_code == 503:
+                text = resp.text
+                logger.warning(f"[tls] 503 {elapsed:.2f}s {full_url}")
+                if "captcha" in text.lower() or "验证码" in text:
+                    raise CaptchaError("触发验证码")
+                raise NetworkError(f"503 Service Unavailable: {url}")
+
+            else:
+                logger.warning(f"[tls] HTTP {resp.status_code} {elapsed:.2f}s {full_url}")
+                raise NetworkError(f"HTTP {resp.status_code}: {url}")
+
+        except (RateLimitError, BlockedError, CaptchaError):
+            raise
+        except Exception as e:
+            elapsed = time.time() - start
+            if "curl_cffi" in str(type(e).__module__):
+                raise NetworkError(f"TLS 请求失败: {e}") from e
+            raise
+
+    async def close(self):
+        if self._session:
+            try:
+                await self._session.close()
+            except Exception:
+                pass
+            self._session = None
+            self._created_at = None
+
+    async def __aenter__(self):
+        await self._ensure_session()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close()