迁移目录

2026-06-12 22:27:18 +08:00
parent 7381097582
commit cd44cd6e47
92 changed files with 1839 additions and 0 deletions
@@ -0,0 +1,148 @@
+"""
+TLS 指纹伪装客户端
+使用 curl_cffi 模拟真实浏览器的 TLS 握手指纹 (JA3/JA4)
+这是绕过懂球帝 TLS 指纹检测的核心模块
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, Optional
+from urllib.parse import urlencode
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import NetworkError, RateLimitError, BlockedError, CaptchaError
+
+
+# curl_cffi 支持的浏览器指纹列表
+IMPERSONATE_TARGETS = [
+    "chrome120", "chrome123", "chrome124",
+    "edge101", "edge99",
+    "safari15_5", "safari17_0",
+]
+
+
+class TLSClient:
+    """
+    基于 curl_cffi 的 TLS 指纹伪装 HTTP 客户端
+    能够模拟真实浏览器的 JA3 指纹，绕过 TLS 指纹检测
+    """
+
+    def __init__(self, impersonate: Optional[str] = None):
+        cfg = get_config()
+        self._impersonate = impersonate or cfg.anti_detect.tls.impersonate
+        self._timeout = cfg.crawler.request_timeout
+        self._session = None
+        self._request_count = 0
+        self._max_per_session = cfg.crawler.session.max_requests_per_session
+        self._created_at = None
+
+    async def _ensure_session(self):
+        if self._session is None or self._should_rotate():
+            await self._rotate_session()
+
+    def _should_rotate(self) -> bool:
+        if self._request_count >= self._max_per_session:
+            return True
+        cfg = get_config().crawler.session
+        if self._created_at and (time.time() - self._created_at) > cfg.rotate_interval:
+            return True
+        return False
+
+    async def _rotate_session(self):
+        await self.close()
+        try:
+            from curl_cffi.requests import AsyncSession
+            target = random.choice(IMPERSONATE_TARGETS) if self._impersonate == "random" else self._impersonate
+            self._session = AsyncSession(impersonate=target, timeout=self._timeout)
+            self._request_count = 0
+            self._created_at = time.time()
+            logger.debug(f"TLS 会话已创建，伪装: {target}")
+        except ImportError:
+            logger.warning("curl_cffi 未安装，回退到 aiohttp（无 TLS 指纹伪装）")
+            self._session = None
+            raise
+
+    async def get(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        proxy: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        await self._ensure_session()
+        self._request_count += 1
+
+        full_url = f"{url}?{urlencode(params)}" if params else url
+        logger.info(f"[tls] GET {full_url}")
+
+        start = time.time()
+        try:
+            resp = await self._session.get(
+                url,
+                params=params,
+                headers=headers,
+                proxy=proxy,
+                allow_redirects=True,
+            )
+            elapsed = time.time() - start
+
+            if resp.status_code == 200:
+                try:
+                    data = resp.json()
+                except Exception:
+                    data = resp.text
+                resp_summary = str(data)[:300] if data else '(empty)'
+                logger.info(f"[tls] 200 OK {elapsed:.2f}s {full_url} resp={resp_summary}")
+                return {
+                    "success": True,
+                    "status": resp.status_code,
+                    "data": data,
+                    "elapsed": elapsed,
+                    "headers": dict(resp.headers),
+                }
+
+            elif resp.status_code == 429:
+                logger.warning(f"[tls] 429 RateLimit {elapsed:.2f}s {full_url}")
+                retry_after = int(resp.headers.get("Retry-After", 60))
+                raise RateLimitError(retry_after=retry_after)
+
+            elif resp.status_code == 403:
+                logger.warning(f"[tls] 403 Forbidden {elapsed:.2f}s {full_url}")
+                raise BlockedError(f"403 Forbidden: {url}")
+
+            elif resp.status_code == 503:
+                text = resp.text
+                logger.warning(f"[tls] 503 {elapsed:.2f}s {full_url}")
+                if "captcha" in text.lower() or "验证码" in text:
+                    raise CaptchaError("触发验证码")
+                raise NetworkError(f"503 Service Unavailable: {url}")
+
+            else:
+                logger.warning(f"[tls] HTTP {resp.status_code} {elapsed:.2f}s {full_url}")
+                raise NetworkError(f"HTTP {resp.status_code}: {url}")
+
+        except (RateLimitError, BlockedError, CaptchaError):
+            raise
+        except Exception as e:
+            elapsed = time.time() - start
+            if "curl_cffi" in str(type(e).__module__):
+                raise NetworkError(f"TLS 请求失败: {e}") from e
+            raise
+
+    async def close(self):
+        if self._session:
+            try:
+                await self._session.close()
+            except Exception:
+                pass
+            self._session = None
+            self._created_at = None
+
+    async def __aenter__(self):
+        await self._ensure_session()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close()