迁移目录
This commit is contained in:
hajimi
@@ -0,0 +1,275 @@
|
||||
"""
|
||||
懂球帝 API 客户端
|
||||
- 支持 aiohttp / curl_cffi 双通道
|
||||
- 自动指纹轮换、频率控制、重试
|
||||
"""
|
||||
import asyncio
|
||||
import random
|
||||
import time
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
|
||||
|
||||
from src.core.config import get_config
|
||||
from src.core.exceptions import (
|
||||
NetworkError, RateLimitError, BlockedError, CaptchaError, ParsingError,
|
||||
)
|
||||
from src.anti_detect.fingerprint import FingerprintPool
|
||||
|
||||
|
||||
class APIClient:
|
||||
"""懂球帝 API 客户端 (aiohttp 通道)"""
|
||||
|
||||
def __init__(self, fingerprint_pool: Optional[FingerprintPool] = None):
|
||||
self._cfg = get_config()
|
||||
self._fp_pool = fingerprint_pool
|
||||
self._session: Optional[aiohttp.ClientSession] = None
|
||||
self._request_count = 0
|
||||
self._created_at: Optional[float] = None
|
||||
self._stats = {
|
||||
"total": 0, "success": 0, "fail": 0,
|
||||
"rate_limit": 0, "blocked": 0, "total_time": 0.0,
|
||||
}
|
||||
|
||||
async def initialize(self):
|
||||
if self._session is None or self._should_rotate():
|
||||
await self._new_session()
|
||||
|
||||
async def _new_session(self):
|
||||
await self.close()
|
||||
timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
|
||||
self._session = aiohttp.ClientSession(timeout=timeout)
|
||||
self._request_count = 0
|
||||
self._created_at = time.time()
|
||||
|
||||
def _should_rotate(self) -> bool:
|
||||
scfg = self._cfg.crawler.session
|
||||
if self._request_count >= scfg.max_requests_per_session:
|
||||
return True
|
||||
if self._created_at and (time.time() - self._created_at) > scfg.rotate_interval:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def close(self):
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
|
||||
async def _get_headers(self) -> tuple:
|
||||
"""获取请求头，返回 (headers, fp_id)"""
|
||||
if self._fp_pool:
|
||||
fp = await self._fp_pool.acquire()
|
||||
return fp.get("http_headers", {}), fp.get("id")
|
||||
|
||||
return {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Referer": "https://www.dongqiudi.com/",
|
||||
"Origin": "https://www.dongqiudi.com",
|
||||
}, None
|
||||
|
||||
async def request(
|
||||
self,
|
||||
url: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
proxy: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
await self.initialize()
|
||||
headers, fp_id = await self._get_headers()
|
||||
self._request_count += 1
|
||||
self._stats["total"] += 1
|
||||
start = time.time()
|
||||
|
||||
full_url = f"{url}?{urlencode(params)}" if params else url
|
||||
logger.info(f"[api] GET {full_url}")
|
||||
|
||||
try:
|
||||
async with self._session.get(
|
||||
url, params=params, headers=headers, proxy=proxy,
|
||||
allow_redirects=True, ssl=False,
|
||||
) as resp:
|
||||
elapsed = time.time() - start
|
||||
self._stats["total_time"] += elapsed
|
||||
return await self._handle_response(resp, url, fp_id, elapsed, params)
|
||||
|
||||
except (RateLimitError, BlockedError, CaptchaError):
|
||||
self._stats["fail"] += 1
|
||||
raise
|
||||
except aiohttp.ClientError as e:
|
||||
self._stats["fail"] += 1
|
||||
if fp_id and self._fp_pool:
|
||||
await self._fp_pool.mark_failure(fp_id, "network")
|
||||
raise NetworkError(f"网络错误: {e}") from e
|
||||
except Exception as e:
|
||||
self._stats["fail"] += 1
|
||||
raise NetworkError(f"请求异常: {e}") from e
|
||||
|
||||
async def _handle_response(
|
||||
self, resp: aiohttp.ClientResponse, url: str, fp_id: Optional[str], elapsed: float,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
status = resp.status
|
||||
full_url = f"{url}?{urlencode(params)}" if params else url
|
||||
|
||||
if status == 200:
|
||||
try:
|
||||
data = await resp.json(content_type=None)
|
||||
except Exception:
|
||||
text = await resp.text()
|
||||
raise ParsingError(f"JSON 解析失败，原始响应: {text[:500]}")
|
||||
|
||||
resp_summary = str(data)[:300] if data else '(empty)'
|
||||
logger.info(f"[api] 200 OK {elapsed:.2f}s {full_url} resp={resp_summary}")
|
||||
self._stats["success"] += 1
|
||||
if fp_id and self._fp_pool:
|
||||
await self._fp_pool.mark_success(fp_id)
|
||||
return data
|
||||
|
||||
elif status == 429:
|
||||
logger.warning(f"[api] 429 RateLimit {elapsed:.2f}s {full_url}")
|
||||
self._stats["rate_limit"] += 1
|
||||
retry_after = int(resp.headers.get("Retry-After", 60))
|
||||
if fp_id and self._fp_pool:
|
||||
await self._fp_pool.mark_failure(fp_id, "rate_limit")
|
||||
raise RateLimitError(retry_after=retry_after)
|
||||
|
||||
elif status == 403:
|
||||
logger.warning(f"[api] 403 Forbidden {elapsed:.2f}s {full_url}")
|
||||
self._stats["blocked"] += 1
|
||||
if fp_id and self._fp_pool:
|
||||
await self._fp_pool.mark_failure(fp_id, "ip_blocked")
|
||||
raise BlockedError(f"403: {url}")
|
||||
|
||||
elif status == 503:
|
||||
text = await resp.text()
|
||||
logger.warning(f"[api] 503 {elapsed:.2f}s {full_url}")
|
||||
if "captcha" in text.lower() or "验证码" in text:
|
||||
if fp_id and self._fp_pool:
|
||||
await self._fp_pool.mark_failure(fp_id, "captcha")
|
||||
raise CaptchaError("验证码")
|
||||
raise NetworkError(f"503: {url}")
|
||||
|
||||
else:
|
||||
logger.warning(f"[api] HTTP {status} {elapsed:.2f}s {full_url}")
|
||||
raise NetworkError(f"HTTP {status}: {url}")
|
||||
|
||||
# ── 懂球帝专用 API 方法 ──
|
||||
|
||||
def _default_params(self, extra: Optional[Dict] = None) -> Dict[str, Any]:
|
||||
dp = self._cfg.dongqiudi.default_params
|
||||
params = {"app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
|
||||
if extra:
|
||||
params.update(extra)
|
||||
return params
|
||||
|
||||
async def get_standings(self, season_id: int) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.standings
|
||||
params = self._default_params({"season_id": season_id})
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_schedule(self, season_id: int, round_number: Optional[int] = None) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.schedule
|
||||
extra = {"season_id": season_id}
|
||||
if round_number:
|
||||
extra["round"] = round_number
|
||||
return await self.request(url, self._default_params(extra))
|
||||
|
||||
async def get_match_detail(self, match_id: int) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.match_detail
|
||||
return await self.request(url, self._default_params({"match_id": match_id}))
|
||||
|
||||
async def get_match_menu(self) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.web_url + self._cfg.dongqiudi.api.match_menu
|
||||
params = {"mark": "gif", "platform": "www", "version": "713"}
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_live_matches(self) -> Dict[str, Any]:
|
||||
"""从各联赛 tab API 汇总当前比赛数据"""
|
||||
all_matches = []
|
||||
league_ids = set()
|
||||
for t in self._cfg.dongqiudi.targets:
|
||||
if not t.active:
|
||||
continue
|
||||
lid = self._get_league_id(t.league)
|
||||
if lid and lid not in league_ids:
|
||||
league_ids.add(lid)
|
||||
try:
|
||||
url = f"https://api.dongqiudi.com/data/tab/league/new/{lid}"
|
||||
raw = await self.request(url, None)
|
||||
items = raw.get("list", []) if isinstance(raw, dict) else []
|
||||
all_matches.extend(items)
|
||||
except Exception as e:
|
||||
logger.warning(f"获取联赛 {t.league} 实时数据失败: {e}")
|
||||
return {"data": {"matches": all_matches}}
|
||||
|
||||
_LEAGUE_ID_MAP = {
|
||||
"中超": 43, "中甲": 129, "亚冠": 226, "英超": 4, "西甲": 3,
|
||||
"德甲": 5, "意甲": 9, "法甲": 12, "欧冠": 6, "欧联": 14,
|
||||
}
|
||||
|
||||
def _get_league_id(self, league_name: str) -> Optional[int]:
|
||||
return self._LEAGUE_ID_MAP.get(league_name)
|
||||
|
||||
async def get_news(self, tab_id: int = 1, size: int = 30) -> Dict[str, Any]:
|
||||
"""通过 tabs API 获取资讯列表"""
|
||||
url = f"{self._cfg.dongqiudi.web_url}/api/app/tabs/web/{tab_id}.json"
|
||||
params = {"size": size} if size != 20 else None
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_video_list(self, tab_id: int = 233) -> Dict[str, Any]:
|
||||
"""抓取视频列表页 HTML，返回 {'html': str, 'tab_id': int}"""
|
||||
await self.initialize()
|
||||
url = f"{self._cfg.dongqiudi.web_url}/videoList/{tab_id}"
|
||||
headers, fp_id = await self._get_headers()
|
||||
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||
self._request_count += 1
|
||||
self._stats["total"] += 1
|
||||
try:
|
||||
async with self._session.get(url, headers=headers, ssl=False) as resp:
|
||||
if resp.status == 200:
|
||||
html = await resp.text()
|
||||
self._stats["success"] += 1
|
||||
return {"html": html, "tab_id": tab_id}
|
||||
raise NetworkError(f"HTTP {resp.status}: {url}")
|
||||
except aiohttp.ClientError as e:
|
||||
self._stats["fail"] += 1
|
||||
raise NetworkError(f"网络错误: {e}") from e
|
||||
|
||||
async def get_article_detail(self, article_id: int) -> Dict[str, Any]:
|
||||
"""抓取文章详情页 HTML，返回 {'html': str}"""
|
||||
await self.initialize()
|
||||
url = f"{self._cfg.dongqiudi.web_url}/articles/{article_id}.html"
|
||||
headers, fp_id = await self._get_headers()
|
||||
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||
self._request_count += 1
|
||||
self._stats["total"] += 1
|
||||
try:
|
||||
async with self._session.get(url, headers=headers, ssl=False) as resp:
|
||||
if resp.status == 200:
|
||||
html = await resp.text()
|
||||
self._stats["success"] += 1
|
||||
return {"html": html, "article_id": article_id}
|
||||
raise NetworkError(f"HTTP {resp.status}: {url}")
|
||||
except aiohttp.ClientError as e:
|
||||
self._stats["fail"] += 1
|
||||
raise NetworkError(f"网络错误: {e}") from e
|
||||
|
||||
@property
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
total = self._stats["total"]
|
||||
return {
|
||||
**self._stats,
|
||||
"success_rate": self._stats["success"] / total if total else 1.0,
|
||||
"avg_time": self._stats["total_time"] / total if total else 0.0,
|
||||
}
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.initialize()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.close()
|
||||
@@ -0,0 +1,497 @@
|
||||
"""
|
||||
混合爬虫引擎
|
||||
- API 优先 → TLS 伪装降级 → Playwright 降级
|
||||
- 智能选择最优通道、自动重试、频率控制
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from enum import Enum
|
||||
from typing import Dict, Any, Optional, List
|
||||
|
||||
import aiohttp
|
||||
from loguru import logger
|
||||
|
||||
from src.core.config import get_config
|
||||
from src.core.error_collector import ErrorCollector
|
||||
from src.core.exceptions import (
|
||||
CrawlerError, RateLimitError, BlockedError, CaptchaError, NetworkError,
|
||||
)
|
||||
from src.anti_detect.fingerprint import FingerprintPool
|
||||
from src.anti_detect.proxy_manager import ProxyManager
|
||||
from src.engine.api_client import APIClient
|
||||
|
||||
|
||||
class Channel(Enum):
|
||||
API = "api"
|
||||
TLS = "tls"
|
||||
PLAYWRIGHT = "playwright"
|
||||
|
||||
|
||||
class HybridEngine:
|
||||
"""混合爬虫引擎"""
|
||||
|
||||
def __init__(self):
|
||||
self._cfg = get_config()
|
||||
self._mode = self._cfg.crawler.mode
|
||||
|
||||
# 组件
|
||||
self._fp_pool = FingerprintPool()
|
||||
self._proxy_mgr = ProxyManager()
|
||||
self._api_client: Optional[APIClient] = None
|
||||
self._tls_client = None
|
||||
self._browser = None
|
||||
|
||||
# 通道统计（用于智能模式选择）
|
||||
self._channel_stats: Dict[str, Dict[str, int]] = {
|
||||
ch.value: {"success": 0, "fail": 0} for ch in Channel
|
||||
}
|
||||
|
||||
self._initialized = False
|
||||
|
||||
async def initialize(self):
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
await self._fp_pool.initialize()
|
||||
self._api_client = APIClient(self._fp_pool)
|
||||
|
||||
# TLS 客户端（可选）
|
||||
if self._cfg.anti_detect.tls.enabled:
|
||||
try:
|
||||
from src.anti_detect.tls_client import TLSClient
|
||||
self._tls_client = TLSClient()
|
||||
logger.info("TLS 伪装客户端已就绪")
|
||||
except Exception as e:
|
||||
logger.warning(f"TLS 客户端初始化失败（curl_cffi 可能未安装）: {e}")
|
||||
|
||||
self._initialized = True
|
||||
logger.info(f"混合引擎初始化完成，模式: {self._mode}")
|
||||
|
||||
async def close(self):
|
||||
if self._api_client:
|
||||
await self._api_client.close()
|
||||
if self._tls_client:
|
||||
await self._tls_client.close()
|
||||
if self._browser:
|
||||
await self._browser.stop()
|
||||
self._initialized = False
|
||||
|
||||
# ── 通用请求方法 ──
|
||||
|
||||
async def request(
|
||||
self,
|
||||
url: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
max_retries: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
统一请求入口，根据模式自动选择通道并降级
|
||||
"""
|
||||
await self.initialize()
|
||||
|
||||
retries = max_retries or self._cfg.crawler.max_retries
|
||||
channels = self._get_channel_order()
|
||||
last_error = None
|
||||
ec = ErrorCollector.get()
|
||||
full_url = url
|
||||
if params:
|
||||
from urllib.parse import urlencode
|
||||
full_url = f"{url}?{urlencode(params)}"
|
||||
params_str = json.dumps(params, ensure_ascii=False) if params else ""
|
||||
|
||||
for attempt in range(retries):
|
||||
for channel in channels:
|
||||
try:
|
||||
await self._smart_delay()
|
||||
proxy = await self._proxy_mgr.get_proxy() if self._proxy_mgr.enabled else None
|
||||
|
||||
result = await self._dispatch(channel, url, params, proxy)
|
||||
self._channel_stats[channel.value]["success"] += 1
|
||||
return result
|
||||
|
||||
except RateLimitError as e:
|
||||
self._channel_stats[channel.value]["fail"] += 1
|
||||
wait = e.retry_after * (1 + random.uniform(0, 0.3))
|
||||
logger.warning(f"[{channel.value}] 频率限制，等待 {wait:.0f}s")
|
||||
await ec.report(request_url=full_url, request_params=params_str, http_status=429, error_type="RateLimitError", error_message=str(e), channel=channel.value)
|
||||
await asyncio.sleep(wait)
|
||||
last_error = e
|
||||
|
||||
except BlockedError as e:
|
||||
self._channel_stats[channel.value]["fail"] += 1
|
||||
logger.warning(f"[{channel.value}] IP 被封，尝试下一通道")
|
||||
await ec.report(request_url=full_url, request_params=params_str, http_status=403, error_type="BlockedError", error_message=str(e), channel=channel.value)
|
||||
last_error = e
|
||||
continue
|
||||
|
||||
except CaptchaError as e:
|
||||
self._channel_stats[channel.value]["fail"] += 1
|
||||
logger.warning(f"[{channel.value}] 触发验证码，降级到 Playwright")
|
||||
await ec.report(request_url=full_url, request_params=params_str, http_status=503, error_type="CaptchaError", error_message=str(e), channel=channel.value)
|
||||
if channel != Channel.PLAYWRIGHT:
|
||||
continue
|
||||
last_error = e
|
||||
|
||||
except (NetworkError, CrawlerError) as e:
|
||||
self._channel_stats[channel.value]["fail"] += 1
|
||||
logger.warning(f"[{channel.value}] 请求失败: {e}")
|
||||
http_code = 0
|
||||
err_msg = str(e)
|
||||
if "404" in err_msg:
|
||||
http_code = 404
|
||||
elif "500" in err_msg:
|
||||
http_code = 500
|
||||
await ec.report(request_url=full_url, request_params=params_str, http_status=http_code, error_type=type(e).__name__, error_message=err_msg, channel=channel.value)
|
||||
last_error = e
|
||||
continue
|
||||
|
||||
# 本轮所有通道都失败，退避后重试
|
||||
backoff = self._cfg.crawler.retry_backoff ** attempt * (1 + random.uniform(0, 0.5))
|
||||
logger.info(f"第 {attempt + 1} 轮全部失败，退避 {backoff:.1f}s 后重试")
|
||||
await asyncio.sleep(backoff)
|
||||
|
||||
raise last_error or CrawlerError(f"所有通道和重试均失败: {url}")
|
||||
|
||||
async def _dispatch(
|
||||
self, channel: Channel, url: str, params: Optional[Dict], proxy: Optional[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""分发到具体通道"""
|
||||
if channel == Channel.API:
|
||||
return await self._api_client.request(url, params, proxy)
|
||||
|
||||
elif channel == Channel.TLS:
|
||||
if not self._tls_client:
|
||||
raise NetworkError("TLS 客户端不可用")
|
||||
fp = await self._fp_pool.acquire()
|
||||
headers = fp.get("http_headers", {})
|
||||
result = await self._tls_client.get(url, params=params, headers=headers, proxy=proxy)
|
||||
if result.get("success"):
|
||||
return result["data"]
|
||||
raise NetworkError(result.get("error", "TLS 请求失败"))
|
||||
|
||||
elif channel == Channel.PLAYWRIGHT:
|
||||
if not self._browser:
|
||||
from src.anti_detect.stealth_browser import StealthBrowser
|
||||
self._browser = StealthBrowser()
|
||||
await self._browser.start()
|
||||
|
||||
full_url = url
|
||||
if params:
|
||||
from urllib.parse import urlencode
|
||||
full_url = f"{url}?{urlencode(params)}"
|
||||
|
||||
logger.info(f"[playwright] GET {full_url}")
|
||||
result = await self._browser.fetch_page(full_url, extract_json=True)
|
||||
if result.get("success") and result.get("json_data"):
|
||||
resp_summary = str(result["json_data"])[:300]
|
||||
logger.info(f"[playwright] 200 OK {full_url} resp={resp_summary}")
|
||||
return result["json_data"]
|
||||
elif result.get("success"):
|
||||
logger.info(f"[playwright] OK (no json) {full_url}")
|
||||
return result
|
||||
logger.warning(f"[playwright] FAIL {full_url} error={result.get('error','')}")
|
||||
raise NetworkError(result.get("error", "Playwright 请求失败"))
|
||||
|
||||
raise CrawlerError(f"未知通道: {channel}")
|
||||
|
||||
def _get_channel_order(self) -> List[Channel]:
|
||||
"""根据模式和历史成功率决定通道优先级"""
|
||||
if self._mode == "api_only":
|
||||
return [Channel.API]
|
||||
elif self._mode == "playwright_only":
|
||||
return [Channel.PLAYWRIGHT]
|
||||
elif self._mode == "hybrid":
|
||||
order = [Channel.API]
|
||||
if self._tls_client:
|
||||
order.append(Channel.TLS)
|
||||
order.append(Channel.PLAYWRIGHT)
|
||||
return order
|
||||
elif self._mode == "smart":
|
||||
return self._smart_channel_order()
|
||||
return [Channel.API, Channel.PLAYWRIGHT]
|
||||
|
||||
def _smart_channel_order(self) -> List[Channel]:
|
||||
"""智能排序：按成功率排序"""
|
||||
def _rate(ch: Channel) -> float:
|
||||
s = self._channel_stats[ch.value]
|
||||
total = s["success"] + s["fail"]
|
||||
return s["success"] / total if total > 5 else 0.5
|
||||
|
||||
candidates = [Channel.API]
|
||||
if self._tls_client:
|
||||
candidates.append(Channel.TLS)
|
||||
candidates.append(Channel.PLAYWRIGHT)
|
||||
|
||||
candidates.sort(key=_rate, reverse=True)
|
||||
return candidates
|
||||
|
||||
async def _smart_delay(self):
|
||||
"""高斯分布随机延迟"""
|
||||
dcfg = self._cfg.crawler.delay
|
||||
delay = random.gauss(dcfg.base, (dcfg.max - dcfg.min) / 4)
|
||||
delay = max(dcfg.min, min(dcfg.max, delay))
|
||||
if dcfg.jitter:
|
||||
delay += random.uniform(-0.5, 0.5)
|
||||
delay = max(0.5, delay)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
# ── 懂球帝专用方法（通过混合引擎转发） ──
|
||||
|
||||
async def get_standings(self, season_id: int) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.standings
|
||||
dp = self._cfg.dongqiudi.default_params
|
||||
params = {"season_id": season_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_schedule(self, season_id: int, round_number: Optional[int] = None) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.schedule
|
||||
dp = self._cfg.dongqiudi.default_params
|
||||
params = {"season_id": season_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
|
||||
if round_number:
|
||||
params["round"] = round_number
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_match_detail(self, match_id: int) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.match_detail
|
||||
dp = self._cfg.dongqiudi.default_params
|
||||
params = {"match_id": match_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_match_situation(self, match_id: int) -> Dict[str, Any]:
|
||||
"""获取比赛实况数据（situation API）"""
|
||||
url = f"https://api.dongqiudi.com{self._cfg.dongqiudi.api.match_situation}/{match_id}"
|
||||
return await self.request(url)
|
||||
|
||||
async def get_match_lineup(self, match_id: int) -> Dict[str, Any]:
|
||||
"""获取比赛阵容数据（lineup API）"""
|
||||
url = f"https://api.dongqiudi.com{self._cfg.dongqiudi.api.match_lineup}/{match_id}"
|
||||
return await self.request(url)
|
||||
|
||||
async def fetch_match_detail_from_page(self, match_id: int) -> Dict[str, Any]:
|
||||
"""从懂球帝移动端页面抓取比赛详情(SSR数据)"""
|
||||
page_url = f"http://m.dongqiudi.com/matchDetail/{match_id}/lotteryOddsNew?cmp_type=soccer"
|
||||
logger.info(f"[page] GET {page_url}")
|
||||
start = time.time()
|
||||
ec = ErrorCollector.get()
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(page_url, headers=headers, ssl=False) as resp:
|
||||
elapsed = time.time() - start
|
||||
if resp.status != 200:
|
||||
logger.warning(f"[page] HTTP {resp.status} {elapsed:.2f}s {page_url}")
|
||||
await ec.report(request_url=page_url, http_status=resp.status, error_type=f"HTTP {resp.status}", error_message=f"页面请求失败 HTTP {resp.status}", channel="page")
|
||||
raise NetworkError(f"页面请求失败 HTTP {resp.status}: {page_url}")
|
||||
html = await resp.text()
|
||||
logger.info(f"[page] 200 OK {elapsed:.2f}s {page_url} html_len={len(html)}")
|
||||
except NetworkError:
|
||||
raise
|
||||
except Exception as e:
|
||||
await ec.report(request_url=page_url, http_status=0, error_type=type(e).__name__, error_message=str(e), channel="page")
|
||||
raise NetworkError(f"页面请求异常: {page_url} {e}") from e
|
||||
|
||||
m = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})\s*;', html, re.DOTALL)
|
||||
if not m:
|
||||
await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="页面中未找到 __INITIAL_STATE__", response_body=html[:2000], channel="page")
|
||||
raise NetworkError(f"页面中未找到 __INITIAL_STATE__: {page_url}")
|
||||
|
||||
state = json.loads(m.group(1))
|
||||
header_data = (state.get("matchContentStore", {})
|
||||
.get("matchDetailHeaderData", {}))
|
||||
if not header_data:
|
||||
await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="matchDetailHeaderData 为空", channel="page")
|
||||
raise NetworkError(f"matchDetailHeaderData 为空: {page_url}")
|
||||
|
||||
match_obj = header_data.get("match", {})
|
||||
info_obj = header_data.get("info", {})
|
||||
logger.info(
|
||||
f"[page] 解析完成: match_id={match_id} "
|
||||
f"status={match_obj.get('status','')} "
|
||||
f"team_A.fs={match_obj.get('team_A',{}).get('fs','')} "
|
||||
f"team_B.fs={match_obj.get('team_B',{}).get('fs','')} "
|
||||
f"events_count={len(info_obj.get('events') or [])}"
|
||||
)
|
||||
return header_data
|
||||
|
||||
async def fetch_match_list_from_page(self, tab_id: int = 5) -> List[Dict[str, Any]]:
|
||||
"""从懂球帝移动端赛程页面抓取 matchListStore.matchList（SSR数据）"""
|
||||
page_url = f"http://m.dongqiudi.com/match/{tab_id}"
|
||||
logger.info(f"[page] GET {page_url}")
|
||||
start = time.time()
|
||||
ec = ErrorCollector.get()
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(page_url, headers=headers, ssl=False) as resp:
|
||||
elapsed = time.time() - start
|
||||
if resp.status != 200:
|
||||
logger.warning(f"[page] HTTP {resp.status} {elapsed:.2f}s {page_url}")
|
||||
await ec.report(request_url=page_url, http_status=resp.status, error_type=f"HTTP {resp.status}", error_message=f"页面请求失败 HTTP {resp.status}", channel="page")
|
||||
raise NetworkError(f"页面请求失败 HTTP {resp.status}: {page_url}")
|
||||
html = await resp.text()
|
||||
logger.info(f"[page] 200 OK {elapsed:.2f}s {page_url} html_len={len(html)}")
|
||||
except NetworkError:
|
||||
raise
|
||||
except Exception as e:
|
||||
await ec.report(request_url=page_url, http_status=0, error_type=type(e).__name__, error_message=str(e), channel="page")
|
||||
raise NetworkError(f"页面请求异常: {page_url} {e}") from e
|
||||
|
||||
m = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})\s*;', html, re.DOTALL)
|
||||
if not m:
|
||||
await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="页面中未找到 __INITIAL_STATE__", response_body=html[:2000], channel="page")
|
||||
raise NetworkError(f"页面中未找到 __INITIAL_STATE__: {page_url}")
|
||||
|
||||
state = json.loads(m.group(1))
|
||||
match_list = state.get("matchListStore", {}).get("matchList", [])
|
||||
logger.info(f"[page] 解析完成: tab_id={tab_id} matchList={len(match_list)} 条")
|
||||
return match_list
|
||||
|
||||
async def get_schedule_round(self, url: str) -> Dict[str, Any]:
|
||||
"""请求具体轮次的赛程URL（含round_id/gameweek参数）"""
|
||||
if 'language=' not in url:
|
||||
url += '&language=zh-cn'
|
||||
return await self.request(url)
|
||||
|
||||
async def get_league_tab_matches(self, api_url: str) -> Dict[str, Any]:
|
||||
"""通过联赛 tab API 获取比赛列表"""
|
||||
return await self.request(api_url)
|
||||
|
||||
async def get_live_text(self, match_id: int, num: int = 200) -> Dict[str, Any]:
|
||||
"""获取比赛文字直播"""
|
||||
url = f"{self._cfg.dongqiudi.web_url}/api/v4/imserver/baidu/Chatroom/rooms"
|
||||
params = {"match_id": match_id, "num": num}
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_live_text_page(self, page_url: str) -> Dict[str, Any]:
|
||||
"""文字直播翻页请求"""
|
||||
return await self.request(page_url)
|
||||
|
||||
async def get_match_menu(self) -> Dict[str, Any]:
|
||||
url = self._cfg.dongqiudi.web_url + self._cfg.dongqiudi.api.match_menu
|
||||
params = {"mark": "gif", "platform": "www", "version": "713"}
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_live_matches(self) -> Dict[str, Any]:
|
||||
"""从各联赛 tab API 汇总当前比赛数据"""
|
||||
all_matches = []
|
||||
league_ids = set()
|
||||
for t in self._cfg.dongqiudi.targets:
|
||||
if not t.active:
|
||||
continue
|
||||
lid = self._LEAGUE_ID_MAP.get(t.league)
|
||||
if lid and lid not in league_ids:
|
||||
league_ids.add(lid)
|
||||
try:
|
||||
url = f"https://api.dongqiudi.com/data/tab/league/new/{lid}"
|
||||
raw = await self.request(url, None)
|
||||
items = raw.get("list", []) if isinstance(raw, dict) else []
|
||||
all_matches.extend(items)
|
||||
except Exception as e:
|
||||
logger.warning(f"获取联赛 {t.league} 实时数据失败: {e}")
|
||||
return {"data": {"matches": all_matches}}
|
||||
|
||||
_LEAGUE_ID_MAP = {
|
||||
"中超": 43, "中甲": 129, "亚冠": 226, "英超": 4, "西甲": 3,
|
||||
"德甲": 5, "意甲": 9, "法甲": 12, "欧冠": 6, "欧联": 14,
|
||||
}
|
||||
|
||||
async def get_news(self, tab_id: int = 1, size: int = 30) -> Dict[str, Any]:
|
||||
url = f"{self._cfg.dongqiudi.web_url}/api/app/tabs/web/{tab_id}.json"
|
||||
params = {"size": size} if size != 20 else None
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_league_news(self, tab_id: int, page: int = 1, after: int = 0) -> Dict[str, Any]:
|
||||
"""从懂球帝 iPhone tabs API 获取联赛资讯（支持分页）"""
|
||||
url = f"https://api.dongqiudi.com/app/tabs/iphone/{tab_id}.json"
|
||||
params = {"mark": "gif", "version": "576", "from": f"tab_{tab_id}"}
|
||||
if after:
|
||||
params["after"] = after
|
||||
params["page"] = page
|
||||
return await self.request(url, params)
|
||||
|
||||
async def get_video_list(self, tab_id: int = 233) -> Dict[str, Any]:
|
||||
await self.initialize()
|
||||
return await self._api_client.get_video_list(tab_id)
|
||||
|
||||
async def get_article_detail(self, article_id: int) -> Dict[str, Any]:
|
||||
await self.initialize()
|
||||
return await self._api_client.get_article_detail(article_id)
|
||||
|
||||
async def get_article_html(self, article_id: int) -> str:
|
||||
"""请求 m.dongqiudi.com 文章页面，返回 HTML 字符串"""
|
||||
import aiohttp
|
||||
url = f"https://m.dongqiudi.com/article/{article_id}.html"
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
|
||||
}
|
||||
timeout = aiohttp.ClientTimeout(total=15)
|
||||
async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status == 200:
|
||||
return await resp.text()
|
||||
raise NetworkError(f"HTTP {resp.status}: {url}")
|
||||
|
||||
async def get_lottery_draw_data(self, url: str) -> dict:
|
||||
"""请求彩种开奖数据API（返回JSON）"""
|
||||
await self.initialize()
|
||||
import aiohttp
|
||||
timeout = aiohttp.ClientTimeout(total=15)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status == 200:
|
||||
return await resp.json()
|
||||
raise NetworkError(f"HTTP {resp.status}: {url}")
|
||||
|
||||
async def get_lottery_result(self, url: str) -> str:
|
||||
"""请求六合彩开奖结果（返回原始文本，自签名证书跳过验证）"""
|
||||
await self.initialize()
|
||||
import aiohttp, ssl
|
||||
ssl_ctx = ssl.create_default_context()
|
||||
ssl_ctx.check_hostname = False
|
||||
ssl_ctx.verify_mode = ssl.CERT_NONE
|
||||
ts = int(time.time() * 1000)
|
||||
full_url = f"{url}?_={ts}"
|
||||
timeout = aiohttp.ClientTimeout(total=15)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(full_url, ssl=ssl_ctx) as resp:
|
||||
if resp.status == 200:
|
||||
raw = await resp.read()
|
||||
# 尝试 utf-8 解码，失败则用 gbk
|
||||
try:
|
||||
return raw.decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
return raw.decode('gbk', errors='replace')
|
||||
raise NetworkError(f"HTTP {resp.status}: {full_url}")
|
||||
|
||||
@property
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"channels": self._channel_stats,
|
||||
"fingerprint": self._fp_pool.stats,
|
||||
"proxy": self._proxy_mgr.stats,
|
||||
"api_client": self._api_client.stats if self._api_client else {},
|
||||
}
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.initialize()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.close()
|
||||
Reference in New Issue
Block a user