163 lines
5.2 KiB
Python
163 lines
5.2 KiB
Python
"""
|
|
代理管理器
|
|
- 代理池维护、健康检测、自动轮换
|
|
- 支持 HTTP / SOCKS5 代理
|
|
"""
|
|
import asyncio
|
|
import random
|
|
import time
|
|
from typing import Dict, Any, List, Optional
|
|
|
|
from loguru import logger
|
|
|
|
from src.core.config import get_config
|
|
from src.core.exceptions import ProxyError
|
|
|
|
|
|
class Proxy:
|
|
__slots__ = (
|
|
"url", "protocol", "host", "port", "username", "password",
|
|
"success_count", "fail_count", "last_used", "last_check",
|
|
"latency", "is_active",
|
|
)
|
|
|
|
def __init__(self, url: str, protocol: str = "http"):
|
|
self.url = url
|
|
self.protocol = protocol
|
|
parts = url.replace("http://", "").replace("https://", "").replace("socks5://", "")
|
|
auth_host = parts.split("@")
|
|
if len(auth_host) == 2:
|
|
auth = auth_host[0].split(":")
|
|
self.username = auth[0]
|
|
self.password = auth[1] if len(auth) > 1 else ""
|
|
host_port = auth_host[1].split(":")
|
|
else:
|
|
self.username = ""
|
|
self.password = ""
|
|
host_port = auth_host[0].split(":")
|
|
self.host = host_port[0]
|
|
self.port = int(host_port[1]) if len(host_port) > 1 else 80
|
|
|
|
self.success_count = 0
|
|
self.fail_count = 0
|
|
self.last_used = 0.0
|
|
self.last_check = 0.0
|
|
self.latency = 0.0
|
|
self.is_active = True
|
|
|
|
@property
|
|
def success_rate(self) -> float:
|
|
total = self.success_count + self.fail_count
|
|
return self.success_count / total if total > 0 else 1.0
|
|
|
|
def mark_success(self, latency: float):
|
|
self.success_count += 1
|
|
self.latency = latency
|
|
self.last_used = time.time()
|
|
|
|
def mark_failure(self):
|
|
self.fail_count += 1
|
|
self.last_used = time.time()
|
|
cfg = get_config().anti_detect.proxy
|
|
if self.fail_count >= cfg.max_failures:
|
|
self.is_active = False
|
|
logger.warning(f"代理 {self.host}:{self.port} 因连续失败被停用")
|
|
|
|
|
|
class ProxyManager:
|
|
"""代理池管理器"""
|
|
|
|
def __init__(self):
|
|
self._proxies: List[Proxy] = []
|
|
self._lock = asyncio.Lock()
|
|
self._enabled = get_config().anti_detect.proxy.enabled
|
|
|
|
@property
|
|
def enabled(self) -> bool:
|
|
return self._enabled and len(self._proxies) > 0
|
|
|
|
async def add_proxy(self, url: str, protocol: str = "http"):
|
|
async with self._lock:
|
|
proxy = Proxy(url, protocol)
|
|
self._proxies.append(proxy)
|
|
logger.info(f"添加代理: {proxy.host}:{proxy.port}")
|
|
|
|
async def add_proxies(self, urls: List[str], protocol: str = "http"):
|
|
for url in urls:
|
|
await self.add_proxy(url, protocol)
|
|
|
|
async def get_proxy(self) -> Optional[str]:
|
|
if not self._enabled:
|
|
return None
|
|
|
|
async with self._lock:
|
|
active = [p for p in self._proxies if p.is_active]
|
|
if not active:
|
|
logger.warning("无可用代理")
|
|
return None
|
|
|
|
active.sort(key=lambda p: (p.success_rate, -p.latency), reverse=True)
|
|
|
|
top_n = max(1, len(active) // 3)
|
|
chosen = random.choice(active[:top_n])
|
|
return chosen.url
|
|
|
|
async def mark_success(self, proxy_url: str, latency: float):
|
|
async with self._lock:
|
|
for p in self._proxies:
|
|
if p.url == proxy_url:
|
|
p.mark_success(latency)
|
|
break
|
|
|
|
async def mark_failure(self, proxy_url: str):
|
|
async with self._lock:
|
|
for p in self._proxies:
|
|
if p.url == proxy_url:
|
|
p.mark_failure()
|
|
break
|
|
|
|
async def health_check(self):
|
|
"""批量检测代理健康状态"""
|
|
import aiohttp
|
|
|
|
test_url = get_config().anti_detect.proxy.test_url
|
|
tasks = []
|
|
|
|
async def _check(proxy: Proxy):
|
|
try:
|
|
start = time.time()
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(
|
|
test_url,
|
|
proxy=proxy.url,
|
|
timeout=aiohttp.ClientTimeout(total=10),
|
|
) as resp:
|
|
if resp.status == 200:
|
|
proxy.latency = time.time() - start
|
|
proxy.is_active = True
|
|
proxy.last_check = time.time()
|
|
else:
|
|
proxy.is_active = False
|
|
except Exception:
|
|
proxy.is_active = False
|
|
|
|
async with self._lock:
|
|
for p in self._proxies:
|
|
tasks.append(_check(p))
|
|
|
|
await asyncio.gather(*tasks, return_exceptions=True)
|
|
active_count = sum(1 for p in self._proxies if p.is_active)
|
|
logger.info(f"代理健康检查完成: {active_count}/{len(self._proxies)} 可用")
|
|
|
|
@property
|
|
def stats(self) -> Dict[str, Any]:
|
|
active = sum(1 for p in self._proxies if p.is_active)
|
|
return {
|
|
"total": len(self._proxies),
|
|
"active": active,
|
|
"avg_latency": (
|
|
sum(p.latency for p in self._proxies if p.is_active) / active
|
|
if active else 0
|
|
),
|
|
}
|