149 lines
5.0 KiB
Python
149 lines
5.0 KiB
Python
"""
|
|
TLS 指纹伪装客户端
|
|
使用 curl_cffi 模拟真实浏览器的 TLS 握手指纹 (JA3/JA4)
|
|
这是绕过懂球帝 TLS 指纹检测的核心模块
|
|
"""
|
|
import asyncio
|
|
import random
|
|
import time
|
|
from typing import Dict, Any, Optional
|
|
from urllib.parse import urlencode
|
|
|
|
from loguru import logger
|
|
|
|
from src.core.config import get_config
|
|
from src.core.exceptions import NetworkError, RateLimitError, BlockedError, CaptchaError
|
|
|
|
|
|
# curl_cffi 支持的浏览器指纹列表
|
|
IMPERSONATE_TARGETS = [
|
|
"chrome120", "chrome123", "chrome124",
|
|
"edge101", "edge99",
|
|
"safari15_5", "safari17_0",
|
|
]
|
|
|
|
|
|
class TLSClient:
|
|
"""
|
|
基于 curl_cffi 的 TLS 指纹伪装 HTTP 客户端
|
|
能够模拟真实浏览器的 JA3 指纹，绕过 TLS 指纹检测
|
|
"""
|
|
|
|
def __init__(self, impersonate: Optional[str] = None):
|
|
cfg = get_config()
|
|
self._impersonate = impersonate or cfg.anti_detect.tls.impersonate
|
|
self._timeout = cfg.crawler.request_timeout
|
|
self._session = None
|
|
self._request_count = 0
|
|
self._max_per_session = cfg.crawler.session.max_requests_per_session
|
|
self._created_at = None
|
|
|
|
async def _ensure_session(self):
|
|
if self._session is None or self._should_rotate():
|
|
await self._rotate_session()
|
|
|
|
def _should_rotate(self) -> bool:
|
|
if self._request_count >= self._max_per_session:
|
|
return True
|
|
cfg = get_config().crawler.session
|
|
if self._created_at and (time.time() - self._created_at) > cfg.rotate_interval:
|
|
return True
|
|
return False
|
|
|
|
async def _rotate_session(self):
|
|
await self.close()
|
|
try:
|
|
from curl_cffi.requests import AsyncSession
|
|
target = random.choice(IMPERSONATE_TARGETS) if self._impersonate == "random" else self._impersonate
|
|
self._session = AsyncSession(impersonate=target, timeout=self._timeout)
|
|
self._request_count = 0
|
|
self._created_at = time.time()
|
|
logger.debug(f"TLS 会话已创建，伪装: {target}")
|
|
except ImportError:
|
|
logger.warning("curl_cffi 未安装，回退到 aiohttp（无 TLS 指纹伪装）")
|
|
self._session = None
|
|
raise
|
|
|
|
async def get(
|
|
self,
|
|
url: str,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
proxy: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
await self._ensure_session()
|
|
self._request_count += 1
|
|
|
|
full_url = f"{url}?{urlencode(params)}" if params else url
|
|
logger.info(f"[tls] GET {full_url}")
|
|
|
|
start = time.time()
|
|
try:
|
|
resp = await self._session.get(
|
|
url,
|
|
params=params,
|
|
headers=headers,
|
|
proxy=proxy,
|
|
allow_redirects=True,
|
|
)
|
|
elapsed = time.time() - start
|
|
|
|
if resp.status_code == 200:
|
|
try:
|
|
data = resp.json()
|
|
except Exception:
|
|
data = resp.text
|
|
resp_summary = str(data)[:300] if data else '(empty)'
|
|
logger.info(f"[tls] 200 OK {elapsed:.2f}s {full_url} resp={resp_summary}")
|
|
return {
|
|
"success": True,
|
|
"status": resp.status_code,
|
|
"data": data,
|
|
"elapsed": elapsed,
|
|
"headers": dict(resp.headers),
|
|
}
|
|
|
|
elif resp.status_code == 429:
|
|
logger.warning(f"[tls] 429 RateLimit {elapsed:.2f}s {full_url}")
|
|
retry_after = int(resp.headers.get("Retry-After", 60))
|
|
raise RateLimitError(retry_after=retry_after)
|
|
|
|
elif resp.status_code == 403:
|
|
logger.warning(f"[tls] 403 Forbidden {elapsed:.2f}s {full_url}")
|
|
raise BlockedError(f"403 Forbidden: {url}")
|
|
|
|
elif resp.status_code == 503:
|
|
text = resp.text
|
|
logger.warning(f"[tls] 503 {elapsed:.2f}s {full_url}")
|
|
if "captcha" in text.lower() or "验证码" in text:
|
|
raise CaptchaError("触发验证码")
|
|
raise NetworkError(f"503 Service Unavailable: {url}")
|
|
|
|
else:
|
|
logger.warning(f"[tls] HTTP {resp.status_code} {elapsed:.2f}s {full_url}")
|
|
raise NetworkError(f"HTTP {resp.status_code}: {url}")
|
|
|
|
except (RateLimitError, BlockedError, CaptchaError):
|
|
raise
|
|
except Exception as e:
|
|
elapsed = time.time() - start
|
|
if "curl_cffi" in str(type(e).__module__):
|
|
raise NetworkError(f"TLS 请求失败: {e}") from e
|
|
raise
|
|
|
|
async def close(self):
|
|
if self._session:
|
|
try:
|
|
await self._session.close()
|
|
except Exception:
|
|
pass
|
|
self._session = None
|
|
self._created_at = None
|
|
|
|
async def __aenter__(self):
|
|
await self._ensure_session()
|
|
return self
|
|
|
|
async def __aexit__(self, *args):
|
|
await self.close()
|