""" TLS 指纹伪装客户端 使用 curl_cffi 模拟真实浏览器的 TLS 握手指纹 (JA3/JA4) 这是绕过懂球帝 TLS 指纹检测的核心模块 """ import asyncio import random import time from typing import Dict, Any, Optional from urllib.parse import urlencode from loguru import logger from src.core.config import get_config from src.core.exceptions import NetworkError, RateLimitError, BlockedError, CaptchaError # curl_cffi 支持的浏览器指纹列表 IMPERSONATE_TARGETS = [ "chrome120", "chrome123", "chrome124", "edge101", "edge99", "safari15_5", "safari17_0", ] class TLSClient: """ 基于 curl_cffi 的 TLS 指纹伪装 HTTP 客户端 能够模拟真实浏览器的 JA3 指纹,绕过 TLS 指纹检测 """ def __init__(self, impersonate: Optional[str] = None): cfg = get_config() self._impersonate = impersonate or cfg.anti_detect.tls.impersonate self._timeout = cfg.crawler.request_timeout self._session = None self._request_count = 0 self._max_per_session = cfg.crawler.session.max_requests_per_session self._created_at = None async def _ensure_session(self): if self._session is None or self._should_rotate(): await self._rotate_session() def _should_rotate(self) -> bool: if self._request_count >= self._max_per_session: return True cfg = get_config().crawler.session if self._created_at and (time.time() - self._created_at) > cfg.rotate_interval: return True return False async def _rotate_session(self): await self.close() try: from curl_cffi.requests import AsyncSession target = random.choice(IMPERSONATE_TARGETS) if self._impersonate == "random" else self._impersonate self._session = AsyncSession(impersonate=target, timeout=self._timeout) self._request_count = 0 self._created_at = time.time() logger.debug(f"TLS 会话已创建,伪装: {target}") except ImportError: logger.warning("curl_cffi 未安装,回退到 aiohttp(无 TLS 指纹伪装)") self._session = None raise async def get( self, url: str, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, proxy: Optional[str] = None, ) -> Dict[str, Any]: await self._ensure_session() self._request_count += 1 full_url = f"{url}?{urlencode(params)}" if params else url logger.info(f"[tls] GET {full_url}") start = time.time() try: resp = await self._session.get( url, params=params, headers=headers, proxy=proxy, allow_redirects=True, ) elapsed = time.time() - start if resp.status_code == 200: try: data = resp.json() except Exception: data = resp.text resp_summary = str(data)[:300] if data else '(empty)' logger.info(f"[tls] 200 OK {elapsed:.2f}s {full_url} resp={resp_summary}") return { "success": True, "status": resp.status_code, "data": data, "elapsed": elapsed, "headers": dict(resp.headers), } elif resp.status_code == 429: logger.warning(f"[tls] 429 RateLimit {elapsed:.2f}s {full_url}") retry_after = int(resp.headers.get("Retry-After", 60)) raise RateLimitError(retry_after=retry_after) elif resp.status_code == 403: logger.warning(f"[tls] 403 Forbidden {elapsed:.2f}s {full_url}") raise BlockedError(f"403 Forbidden: {url}") elif resp.status_code == 503: text = resp.text logger.warning(f"[tls] 503 {elapsed:.2f}s {full_url}") if "captcha" in text.lower() or "验证码" in text: raise CaptchaError("触发验证码") raise NetworkError(f"503 Service Unavailable: {url}") else: logger.warning(f"[tls] HTTP {resp.status_code} {elapsed:.2f}s {full_url}") raise NetworkError(f"HTTP {resp.status_code}: {url}") except (RateLimitError, BlockedError, CaptchaError): raise except Exception as e: elapsed = time.time() - start if "curl_cffi" in str(type(e).__module__): raise NetworkError(f"TLS 请求失败: {e}") from e raise async def close(self): if self._session: try: await self._session.close() except Exception: pass self._session = None self._created_at = None async def __aenter__(self): await self._ensure_session() return self async def __aexit__(self, *args): await self.close()