迁移目录

2026-06-12 22:27:18 +08:00
parent 7381097582
commit cd44cd6e47
92 changed files with 1839 additions and 0 deletions
@@ -0,0 +1,497 @@
+"""
+混合爬虫引擎
+- API 优先 → TLS 伪装降级 → Playwright 降级
+- 智能选择最优通道、自动重试、频率控制
+"""
+import asyncio
+import json
+import random
+import re
+import time
+from enum import Enum
+from typing import Dict, Any, Optional, List
+
+import aiohttp
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.error_collector import ErrorCollector
+from src.core.exceptions import (
+    CrawlerError, RateLimitError, BlockedError, CaptchaError, NetworkError,
+)
+from src.anti_detect.fingerprint import FingerprintPool
+from src.anti_detect.proxy_manager import ProxyManager
+from src.engine.api_client import APIClient
+
+
+class Channel(Enum):
+    API = "api"
+    TLS = "tls"
+    PLAYWRIGHT = "playwright"
+
+
+class HybridEngine:
+    """混合爬虫引擎"""
+
+    def __init__(self):
+        self._cfg = get_config()
+        self._mode = self._cfg.crawler.mode
+
+        # 组件
+        self._fp_pool = FingerprintPool()
+        self._proxy_mgr = ProxyManager()
+        self._api_client: Optional[APIClient] = None
+        self._tls_client = None
+        self._browser = None
+
+        # 通道统计（用于智能模式选择）
+        self._channel_stats: Dict[str, Dict[str, int]] = {
+            ch.value: {"success": 0, "fail": 0} for ch in Channel
+        }
+
+        self._initialized = False
+
+    async def initialize(self):
+        if self._initialized:
+            return
+
+        await self._fp_pool.initialize()
+        self._api_client = APIClient(self._fp_pool)
+
+        # TLS 客户端（可选）
+        if self._cfg.anti_detect.tls.enabled:
+            try:
+                from src.anti_detect.tls_client import TLSClient
+                self._tls_client = TLSClient()
+                logger.info("TLS 伪装客户端已就绪")
+            except Exception as e:
+                logger.warning(f"TLS 客户端初始化失败（curl_cffi 可能未安装）: {e}")
+
+        self._initialized = True
+        logger.info(f"混合引擎初始化完成，模式: {self._mode}")
+
+    async def close(self):
+        if self._api_client:
+            await self._api_client.close()
+        if self._tls_client:
+            await self._tls_client.close()
+        if self._browser:
+            await self._browser.stop()
+        self._initialized = False
+
+    # ── 通用请求方法 ──
+
+    async def request(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        max_retries: Optional[int] = None,
+    ) -> Dict[str, Any]:
+        """
+        统一请求入口，根据模式自动选择通道并降级
+        """
+        await self.initialize()
+
+        retries = max_retries or self._cfg.crawler.max_retries
+        channels = self._get_channel_order()
+        last_error = None
+        ec = ErrorCollector.get()
+        full_url = url
+        if params:
+            from urllib.parse import urlencode
+            full_url = f"{url}?{urlencode(params)}"
+        params_str = json.dumps(params, ensure_ascii=False) if params else ""
+
+        for attempt in range(retries):
+            for channel in channels:
+                try:
+                    await self._smart_delay()
+                    proxy = await self._proxy_mgr.get_proxy() if self._proxy_mgr.enabled else None
+
+                    result = await self._dispatch(channel, url, params, proxy)
+                    self._channel_stats[channel.value]["success"] += 1
+                    return result
+
+                except RateLimitError as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    wait = e.retry_after * (1 + random.uniform(0, 0.3))
+                    logger.warning(f"[{channel.value}] 频率限制，等待 {wait:.0f}s")
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=429, error_type="RateLimitError", error_message=str(e), channel=channel.value)
+                    await asyncio.sleep(wait)
+                    last_error = e
+
+                except BlockedError as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    logger.warning(f"[{channel.value}] IP 被封，尝试下一通道")
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=403, error_type="BlockedError", error_message=str(e), channel=channel.value)
+                    last_error = e
+                    continue
+
+                except CaptchaError as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    logger.warning(f"[{channel.value}] 触发验证码，降级到 Playwright")
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=503, error_type="CaptchaError", error_message=str(e), channel=channel.value)
+                    if channel != Channel.PLAYWRIGHT:
+                        continue
+                    last_error = e
+
+                except (NetworkError, CrawlerError) as e:
+                    self._channel_stats[channel.value]["fail"] += 1
+                    logger.warning(f"[{channel.value}] 请求失败: {e}")
+                    http_code = 0
+                    err_msg = str(e)
+                    if "404" in err_msg:
+                        http_code = 404
+                    elif "500" in err_msg:
+                        http_code = 500
+                    await ec.report(request_url=full_url, request_params=params_str, http_status=http_code, error_type=type(e).__name__, error_message=err_msg, channel=channel.value)
+                    last_error = e
+                    continue
+
+            # 本轮所有通道都失败，退避后重试
+            backoff = self._cfg.crawler.retry_backoff ** attempt * (1 + random.uniform(0, 0.5))
+            logger.info(f"第 {attempt + 1} 轮全部失败，退避 {backoff:.1f}s 后重试")
+            await asyncio.sleep(backoff)
+
+        raise last_error or CrawlerError(f"所有通道和重试均失败: {url}")
+
+    async def _dispatch(
+        self, channel: Channel, url: str, params: Optional[Dict], proxy: Optional[str],
+    ) -> Dict[str, Any]:
+        """分发到具体通道"""
+        if channel == Channel.API:
+            return await self._api_client.request(url, params, proxy)
+
+        elif channel == Channel.TLS:
+            if not self._tls_client:
+                raise NetworkError("TLS 客户端不可用")
+            fp = await self._fp_pool.acquire()
+            headers = fp.get("http_headers", {})
+            result = await self._tls_client.get(url, params=params, headers=headers, proxy=proxy)
+            if result.get("success"):
+                return result["data"]
+            raise NetworkError(result.get("error", "TLS 请求失败"))
+
+        elif channel == Channel.PLAYWRIGHT:
+            if not self._browser:
+                from src.anti_detect.stealth_browser import StealthBrowser
+                self._browser = StealthBrowser()
+                await self._browser.start()
+
+            full_url = url
+            if params:
+                from urllib.parse import urlencode
+                full_url = f"{url}?{urlencode(params)}"
+
+            logger.info(f"[playwright] GET {full_url}")
+            result = await self._browser.fetch_page(full_url, extract_json=True)
+            if result.get("success") and result.get("json_data"):
+                resp_summary = str(result["json_data"])[:300]
+                logger.info(f"[playwright] 200 OK {full_url} resp={resp_summary}")
+                return result["json_data"]
+            elif result.get("success"):
+                logger.info(f"[playwright] OK (no json) {full_url}")
+                return result
+            logger.warning(f"[playwright] FAIL {full_url} error={result.get('error','')}")
+            raise NetworkError(result.get("error", "Playwright 请求失败"))
+
+        raise CrawlerError(f"未知通道: {channel}")
+
+    def _get_channel_order(self) -> List[Channel]:
+        """根据模式和历史成功率决定通道优先级"""
+        if self._mode == "api_only":
+            return [Channel.API]
+        elif self._mode == "playwright_only":
+            return [Channel.PLAYWRIGHT]
+        elif self._mode == "hybrid":
+            order = [Channel.API]
+            if self._tls_client:
+                order.append(Channel.TLS)
+            order.append(Channel.PLAYWRIGHT)
+            return order
+        elif self._mode == "smart":
+            return self._smart_channel_order()
+        return [Channel.API, Channel.PLAYWRIGHT]
+
+    def _smart_channel_order(self) -> List[Channel]:
+        """智能排序：按成功率排序"""
+        def _rate(ch: Channel) -> float:
+            s = self._channel_stats[ch.value]
+            total = s["success"] + s["fail"]
+            return s["success"] / total if total > 5 else 0.5
+
+        candidates = [Channel.API]
+        if self._tls_client:
+            candidates.append(Channel.TLS)
+        candidates.append(Channel.PLAYWRIGHT)
+
+        candidates.sort(key=_rate, reverse=True)
+        return candidates
+
+    async def _smart_delay(self):
+        """高斯分布随机延迟"""
+        dcfg = self._cfg.crawler.delay
+        delay = random.gauss(dcfg.base, (dcfg.max - dcfg.min) / 4)
+        delay = max(dcfg.min, min(dcfg.max, delay))
+        if dcfg.jitter:
+            delay += random.uniform(-0.5, 0.5)
+            delay = max(0.5, delay)
+        await asyncio.sleep(delay)
+
+    # ── 懂球帝专用方法（通过混合引擎转发） ──
+
+    async def get_standings(self, season_id: int) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.standings
+        dp = self._cfg.dongqiudi.default_params
+        params = {"season_id": season_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
+        return await self.request(url, params)
+
+    async def get_schedule(self, season_id: int, round_number: Optional[int] = None) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.schedule
+        dp = self._cfg.dongqiudi.default_params
+        params = {"season_id": season_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
+        if round_number:
+            params["round"] = round_number
+        return await self.request(url, params)
+
+    async def get_match_detail(self, match_id: int) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.base_url + self._cfg.dongqiudi.api.match_detail
+        dp = self._cfg.dongqiudi.default_params
+        params = {"match_id": match_id, "app": dp.app, "platform": dp.platform, "version": dp.version, "language": dp.language}
+        return await self.request(url, params)
+
+    async def get_match_situation(self, match_id: int) -> Dict[str, Any]:
+        """获取比赛实况数据（situation API）"""
+        url = f"https://api.dongqiudi.com{self._cfg.dongqiudi.api.match_situation}/{match_id}"
+        return await self.request(url)
+
+    async def get_match_lineup(self, match_id: int) -> Dict[str, Any]:
+        """获取比赛阵容数据（lineup API）"""
+        url = f"https://api.dongqiudi.com{self._cfg.dongqiudi.api.match_lineup}/{match_id}"
+        return await self.request(url)
+
+    async def fetch_match_detail_from_page(self, match_id: int) -> Dict[str, Any]:
+        """从懂球帝移动端页面抓取比赛详情(SSR数据)"""
+        page_url = f"http://m.dongqiudi.com/matchDetail/{match_id}/lotteryOddsNew?cmp_type=soccer"
+        logger.info(f"[page] GET {page_url}")
+        start = time.time()
+        ec = ErrorCollector.get()
+
+        timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
+        headers = {
+            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) "
+                          "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+        }
+        try:
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(page_url, headers=headers, ssl=False) as resp:
+                    elapsed = time.time() - start
+                    if resp.status != 200:
+                        logger.warning(f"[page] HTTP {resp.status} {elapsed:.2f}s {page_url}")
+                        await ec.report(request_url=page_url, http_status=resp.status, error_type=f"HTTP {resp.status}", error_message=f"页面请求失败 HTTP {resp.status}", channel="page")
+                        raise NetworkError(f"页面请求失败 HTTP {resp.status}: {page_url}")
+                    html = await resp.text()
+                    logger.info(f"[page] 200 OK {elapsed:.2f}s {page_url} html_len={len(html)}")
+        except NetworkError:
+            raise
+        except Exception as e:
+            await ec.report(request_url=page_url, http_status=0, error_type=type(e).__name__, error_message=str(e), channel="page")
+            raise NetworkError(f"页面请求异常: {page_url} {e}") from e
+
+        m = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})\s*;', html, re.DOTALL)
+        if not m:
+            await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="页面中未找到 __INITIAL_STATE__", response_body=html[:2000], channel="page")
+            raise NetworkError(f"页面中未找到 __INITIAL_STATE__: {page_url}")
+
+        state = json.loads(m.group(1))
+        header_data = (state.get("matchContentStore", {})
+                            .get("matchDetailHeaderData", {}))
+        if not header_data:
+            await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="matchDetailHeaderData 为空", channel="page")
+            raise NetworkError(f"matchDetailHeaderData 为空: {page_url}")
+
+        match_obj = header_data.get("match", {})
+        info_obj = header_data.get("info", {})
+        logger.info(
+            f"[page] 解析完成: match_id={match_id} "
+            f"status={match_obj.get('status','')} "
+            f"team_A.fs={match_obj.get('team_A',{}).get('fs','')} "
+            f"team_B.fs={match_obj.get('team_B',{}).get('fs','')} "
+            f"events_count={len(info_obj.get('events') or [])}"
+        )
+        return header_data
+
+    async def fetch_match_list_from_page(self, tab_id: int = 5) -> List[Dict[str, Any]]:
+        """从懂球帝移动端赛程页面抓取 matchListStore.matchList（SSR数据）"""
+        page_url = f"http://m.dongqiudi.com/match/{tab_id}"
+        logger.info(f"[page] GET {page_url}")
+        start = time.time()
+        ec = ErrorCollector.get()
+
+        timeout = aiohttp.ClientTimeout(total=self._cfg.crawler.request_timeout)
+        headers = {
+            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) "
+                          "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "zh-CN,zh;q=0.9",
+        }
+        try:
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(page_url, headers=headers, ssl=False) as resp:
+                    elapsed = time.time() - start
+                    if resp.status != 200:
+                        logger.warning(f"[page] HTTP {resp.status} {elapsed:.2f}s {page_url}")
+                        await ec.report(request_url=page_url, http_status=resp.status, error_type=f"HTTP {resp.status}", error_message=f"页面请求失败 HTTP {resp.status}", channel="page")
+                        raise NetworkError(f"页面请求失败 HTTP {resp.status}: {page_url}")
+                    html = await resp.text()
+                    logger.info(f"[page] 200 OK {elapsed:.2f}s {page_url} html_len={len(html)}")
+        except NetworkError:
+            raise
+        except Exception as e:
+            await ec.report(request_url=page_url, http_status=0, error_type=type(e).__name__, error_message=str(e), channel="page")
+            raise NetworkError(f"页面请求异常: {page_url} {e}") from e
+
+        m = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})\s*;', html, re.DOTALL)
+        if not m:
+            await ec.report(request_url=page_url, http_status=200, error_type="ParseError", error_message="页面中未找到 __INITIAL_STATE__", response_body=html[:2000], channel="page")
+            raise NetworkError(f"页面中未找到 __INITIAL_STATE__: {page_url}")
+
+        state = json.loads(m.group(1))
+        match_list = state.get("matchListStore", {}).get("matchList", [])
+        logger.info(f"[page] 解析完成: tab_id={tab_id} matchList={len(match_list)} 条")
+        return match_list
+
+    async def get_schedule_round(self, url: str) -> Dict[str, Any]:
+        """请求具体轮次的赛程URL（含round_id/gameweek参数）"""
+        if 'language=' not in url:
+            url += '&language=zh-cn'
+        return await self.request(url)
+
+    async def get_league_tab_matches(self, api_url: str) -> Dict[str, Any]:
+        """通过联赛 tab API 获取比赛列表"""
+        return await self.request(api_url)
+
+    async def get_live_text(self, match_id: int, num: int = 200) -> Dict[str, Any]:
+        """获取比赛文字直播"""
+        url = f"{self._cfg.dongqiudi.web_url}/api/v4/imserver/baidu/Chatroom/rooms"
+        params = {"match_id": match_id, "num": num}
+        return await self.request(url, params)
+
+    async def get_live_text_page(self, page_url: str) -> Dict[str, Any]:
+        """文字直播翻页请求"""
+        return await self.request(page_url)
+
+    async def get_match_menu(self) -> Dict[str, Any]:
+        url = self._cfg.dongqiudi.web_url + self._cfg.dongqiudi.api.match_menu
+        params = {"mark": "gif", "platform": "www", "version": "713"}
+        return await self.request(url, params)
+
+    async def get_live_matches(self) -> Dict[str, Any]:
+        """从各联赛 tab API 汇总当前比赛数据"""
+        all_matches = []
+        league_ids = set()
+        for t in self._cfg.dongqiudi.targets:
+            if not t.active:
+                continue
+            lid = self._LEAGUE_ID_MAP.get(t.league)
+            if lid and lid not in league_ids:
+                league_ids.add(lid)
+                try:
+                    url = f"https://api.dongqiudi.com/data/tab/league/new/{lid}"
+                    raw = await self.request(url, None)
+                    items = raw.get("list", []) if isinstance(raw, dict) else []
+                    all_matches.extend(items)
+                except Exception as e:
+                    logger.warning(f"获取联赛 {t.league} 实时数据失败: {e}")
+        return {"data": {"matches": all_matches}}
+
+    _LEAGUE_ID_MAP = {
+        "中超": 43, "中甲": 129, "亚冠": 226, "英超": 4, "西甲": 3,
+        "德甲": 5, "意甲": 9, "法甲": 12, "欧冠": 6, "欧联": 14,
+    }
+
+    async def get_news(self, tab_id: int = 1, size: int = 30) -> Dict[str, Any]:
+        url = f"{self._cfg.dongqiudi.web_url}/api/app/tabs/web/{tab_id}.json"
+        params = {"size": size} if size != 20 else None
+        return await self.request(url, params)
+
+    async def get_league_news(self, tab_id: int, page: int = 1, after: int = 0) -> Dict[str, Any]:
+        """从懂球帝 iPhone tabs API 获取联赛资讯（支持分页）"""
+        url = f"https://api.dongqiudi.com/app/tabs/iphone/{tab_id}.json"
+        params = {"mark": "gif", "version": "576", "from": f"tab_{tab_id}"}
+        if after:
+            params["after"] = after
+            params["page"] = page
+        return await self.request(url, params)
+
+    async def get_video_list(self, tab_id: int = 233) -> Dict[str, Any]:
+        await self.initialize()
+        return await self._api_client.get_video_list(tab_id)
+
+    async def get_article_detail(self, article_id: int) -> Dict[str, Any]:
+        await self.initialize()
+        return await self._api_client.get_article_detail(article_id)
+
+    async def get_article_html(self, article_id: int) -> str:
+        """请求 m.dongqiudi.com 文章页面，返回 HTML 字符串"""
+        import aiohttp
+        url = f"https://m.dongqiudi.com/article/{article_id}.html"
+        headers = {
+            "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
+        }
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
+            async with session.get(url) as resp:
+                if resp.status == 200:
+                    return await resp.text()
+                raise NetworkError(f"HTTP {resp.status}: {url}")
+
+    async def get_lottery_draw_data(self, url: str) -> dict:
+        """请求彩种开奖数据API（返回JSON）"""
+        await self.initialize()
+        import aiohttp
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(url) as resp:
+                if resp.status == 200:
+                    return await resp.json()
+                raise NetworkError(f"HTTP {resp.status}: {url}")
+
+    async def get_lottery_result(self, url: str) -> str:
+        """请求六合彩开奖结果（返回原始文本，自签名证书跳过验证）"""
+        await self.initialize()
+        import aiohttp, ssl
+        ssl_ctx = ssl.create_default_context()
+        ssl_ctx.check_hostname = False
+        ssl_ctx.verify_mode = ssl.CERT_NONE
+        ts = int(time.time() * 1000)
+        full_url = f"{url}?_={ts}"
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(full_url, ssl=ssl_ctx) as resp:
+                if resp.status == 200:
+                    raw = await resp.read()
+                    # 尝试 utf-8 解码，失败则用 gbk
+                    try:
+                        return raw.decode('utf-8')
+                    except UnicodeDecodeError:
+                        return raw.decode('gbk', errors='replace')
+                raise NetworkError(f"HTTP {resp.status}: {full_url}")
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        return {
+            "channels": self._channel_stats,
+            "fingerprint": self._fp_pool.stats,
+            "proxy": self._proxy_mgr.stats,
+            "api_client": self._api_client.stats if self._api_client else {},
+        }
+
+    async def __aenter__(self):
+        await self.initialize()
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close()