43 lines
915 B
Plaintext
43 lines
915 B
Plaintext
# 懂球帝爬虫项目依赖
|
|
# HTTP 请求
|
|
aiohttp>=3.9.0
|
|
requests>=2.31.0
|
|
httpx>=0.27.0
|
|
curl_cffi>=0.7.0 # TLS 指纹模拟 (绕过 JA3 检测)
|
|
|
|
# 浏览器自动化
|
|
playwright>=1.42.0
|
|
playwright-stealth>=1.0.6 # Playwright 反检测
|
|
|
|
# 反爬绕过
|
|
fake-useragent>=1.5.0
|
|
cloudscraper>=1.2.71 # 绕过 Cloudflare
|
|
undetected-chromedriver>=3.5.5
|
|
scrapling>=0.4.9 # 动态网页正文抓取
|
|
browserforge>=1.2.4 # scrapling 静态抓取指纹依赖
|
|
|
|
# 代理管理
|
|
PySocks>=1.7.1
|
|
|
|
# 数据处理
|
|
pydantic>=2.6.0
|
|
pydantic-settings>=2.2.0
|
|
PyYAML>=6.0.1
|
|
|
|
# 数据库
|
|
pymysql>=1.1.0
|
|
aiomysql>=0.2.0
|
|
redis>=5.0.0
|
|
aioredis>=2.0.0
|
|
|
|
# 调度
|
|
APScheduler>=3.10.4
|
|
|
|
# 工具
|
|
loguru>=0.7.2 # 高级日志
|
|
tenacity>=8.2.0 # 重试机制
|
|
python-dotenv>=1.0.0
|
|
orjson>=3.9.0 # 高性能 JSON
|
|
beautifulsoup4>=4.12.0 # requests 回退抓取时解析 HTML
|
|
|