迁移目录

docker/crawler/src/anti_detect/proxy_manager.py

@@ -0,0 +1,162 @@
+"""
+代理管理器
+- 代理池维护、健康检测、自动轮换
+- 支持 HTTP / SOCKS5 代理
+"""
+import asyncio
+import random
+import time
+from typing import Dict, Any, List, Optional
+
+from loguru import logger
+
+from src.core.config import get_config
+from src.core.exceptions import ProxyError
+
+
+class Proxy:
+    __slots__ = (
+        "url", "protocol", "host", "port", "username", "password",
+        "success_count", "fail_count", "last_used", "last_check",
+        "latency", "is_active",
+    )
+
+    def __init__(self, url: str, protocol: str = "http"):
+        self.url = url
+        self.protocol = protocol
+        parts = url.replace("http://", "").replace("https://", "").replace("socks5://", "")
+        auth_host = parts.split("@")
+        if len(auth_host) == 2:
+            auth = auth_host[0].split(":")
+            self.username = auth[0]
+            self.password = auth[1] if len(auth) > 1 else ""
+            host_port = auth_host[1].split(":")
+        else:
+            self.username = ""
+            self.password = ""
+            host_port = auth_host[0].split(":")
+        self.host = host_port[0]
+        self.port = int(host_port[1]) if len(host_port) > 1 else 80
+
+        self.success_count = 0
+        self.fail_count = 0
+        self.last_used = 0.0
+        self.last_check = 0.0
+        self.latency = 0.0
+        self.is_active = True
+
+    @property
+    def success_rate(self) -> float:
+        total = self.success_count + self.fail_count
+        return self.success_count / total if total > 0 else 1.0
+
+    def mark_success(self, latency: float):
+        self.success_count += 1
+        self.latency = latency
+        self.last_used = time.time()
+
+    def mark_failure(self):
+        self.fail_count += 1
+        self.last_used = time.time()
+        cfg = get_config().anti_detect.proxy
+        if self.fail_count >= cfg.max_failures:
+            self.is_active = False
+            logger.warning(f"代理 {self.host}:{self.port} 因连续失败被停用")
+
+
+class ProxyManager:
+    """代理池管理器"""
+
+    def __init__(self):
+        self._proxies: List[Proxy] = []
+        self._lock = asyncio.Lock()
+        self._enabled = get_config().anti_detect.proxy.enabled
+
+    @property
+    def enabled(self) -> bool:
+        return self._enabled and len(self._proxies) > 0
+
+    async def add_proxy(self, url: str, protocol: str = "http"):
+        async with self._lock:
+            proxy = Proxy(url, protocol)
+            self._proxies.append(proxy)
+            logger.info(f"添加代理: {proxy.host}:{proxy.port}")
+
+    async def add_proxies(self, urls: List[str], protocol: str = "http"):
+        for url in urls:
+            await self.add_proxy(url, protocol)
+
+    async def get_proxy(self) -> Optional[str]:
+        if not self._enabled:
+            return None
+
+        async with self._lock:
+            active = [p for p in self._proxies if p.is_active]
+            if not active:
+                logger.warning("无可用代理")
+                return None
+
+            active.sort(key=lambda p: (p.success_rate, -p.latency), reverse=True)
+
+            top_n = max(1, len(active) // 3)
+            chosen = random.choice(active[:top_n])
+            return chosen.url
+
+    async def mark_success(self, proxy_url: str, latency: float):
+        async with self._lock:
+            for p in self._proxies:
+                if p.url == proxy_url:
+                    p.mark_success(latency)
+                    break
+
+    async def mark_failure(self, proxy_url: str):
+        async with self._lock:
+            for p in self._proxies:
+                if p.url == proxy_url:
+                    p.mark_failure()
+                    break
+
+    async def health_check(self):
+        """批量检测代理健康状态"""
+        import aiohttp
+
+        test_url = get_config().anti_detect.proxy.test_url
+        tasks = []
+
+        async def _check(proxy: Proxy):
+            try:
+                start = time.time()
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(
+                        test_url,
+                        proxy=proxy.url,
+                        timeout=aiohttp.ClientTimeout(total=10),
+                    ) as resp:
+                        if resp.status == 200:
+                            proxy.latency = time.time() - start
+                            proxy.is_active = True
+                            proxy.last_check = time.time()
+                        else:
+                            proxy.is_active = False
+            except Exception:
+                proxy.is_active = False
+
+        async with self._lock:
+            for p in self._proxies:
+                tasks.append(_check(p))
+
+        await asyncio.gather(*tasks, return_exceptions=True)
+        active_count = sum(1 for p in self._proxies if p.is_active)
+        logger.info(f"代理健康检查完成: {active_count}/{len(self._proxies)} 可用")
+
+    @property
+    def stats(self) -> Dict[str, Any]:
+        active = sum(1 for p in self._proxies if p.is_active)
+        return {
+            "total": len(self._proxies),
+            "active": active,
+            "avg_latency": (
+                sum(p.latency for p in self._proxies if p.is_active) / active
+                if active else 0
+            ),
+        }