184 lines
4.1 KiB
YAML
184 lines
4.1 KiB
YAML
app:
|
|
name: "dongqiudi-crawler"
|
|
version: "1.0.0"
|
|
env: "development"
|
|
|
|
# 数据库配置
|
|
database:
|
|
host: "127.0.0.1"
|
|
port: 3300
|
|
database: "sbnews"
|
|
username: "sbnews"
|
|
password: "72m931X6K5eeWMpW"
|
|
charset: "utf8mb4"
|
|
prefix: "la_"
|
|
pool_size: 10
|
|
|
|
# Redis 配置
|
|
redis:
|
|
host: "127.0.0.1"
|
|
port: 6377
|
|
password: "hajiminanbeilvdou"
|
|
db: 4
|
|
|
|
# 爬虫引擎配置
|
|
crawler:
|
|
mode: "hybrid" # api_only / playwright_only / hybrid / smart
|
|
max_concurrent: 3
|
|
request_timeout: 30
|
|
max_retries: 3
|
|
retry_backoff: 2.0
|
|
|
|
# 延迟策略 (高斯分布)
|
|
delay:
|
|
base: 3.0
|
|
min: 2.0
|
|
max: 8.0
|
|
jitter: true # 随机抖动
|
|
|
|
# 会话管理
|
|
session:
|
|
rotate_interval: 300 # 秒，每5分钟轮换会话
|
|
max_requests_per_session: 50
|
|
|
|
# 反检测配置
|
|
anti_detect:
|
|
# 指纹池
|
|
fingerprint:
|
|
pool_size: 50
|
|
min_success_rate: 0.6
|
|
rotation_interval: 300 # 秒
|
|
cooldown: 60 # 使用后冷却时间
|
|
|
|
# TLS 指纹伪装 (curl_cffi)
|
|
tls:
|
|
enabled: true
|
|
impersonate: "chrome124" # 模拟 Chrome 124 的 TLS 指纹
|
|
|
|
# Playwright 反检测
|
|
playwright:
|
|
browser: "chromium"
|
|
headless: true
|
|
stealth: true
|
|
viewport:
|
|
width: 1920
|
|
height: 1080
|
|
anti_features:
|
|
remove_webdriver: true
|
|
fake_plugins: true
|
|
fake_languages: true
|
|
fake_chrome_runtime: true
|
|
disable_webrtc: true
|
|
random_viewport: true
|
|
human_mouse: true
|
|
random_scroll: true
|
|
|
|
# 代理配置
|
|
proxy:
|
|
enabled: false
|
|
rotate: true
|
|
providers: [] # 代理源列表
|
|
test_url: "https://httpbin.org/ip"
|
|
max_failures: 3
|
|
|
|
# 懂球帝 API 端点
|
|
dongqiudi:
|
|
base_url: "https://sport-data.dongqiudi.com"
|
|
web_url: "https://www.dongqiudi.com"
|
|
|
|
api:
|
|
standings: "/soccer/biz/data/standing"
|
|
schedule: "/soccer/biz/data/schedule"
|
|
match_detail: "/soccer/biz/data/match_detail"
|
|
match_menu: "/api/v2/config/match_menu"
|
|
team_info: "/soccer/biz/data/team_info"
|
|
player_stats: "/soccer/biz/data/player_stats"
|
|
live_matches: "/soccer/biz/data/live"
|
|
news: "/api/v2/article/list"
|
|
|
|
# 默认请求参数
|
|
default_params:
|
|
app: "dqd"
|
|
platform: "www"
|
|
version: "0"
|
|
language: "zh-cn"
|
|
|
|
# 采集目标联赛
|
|
targets:
|
|
- league: "中超"
|
|
league_code: "CSL"
|
|
season_id: 26322
|
|
priority: 1
|
|
active: true
|
|
- league: "中甲"
|
|
league_code: "CL1"
|
|
season_id: 26323
|
|
priority: 2
|
|
active: true
|
|
- league: "亚冠"
|
|
league_code: "ACL"
|
|
season_id: 26324
|
|
priority: 3
|
|
active: true
|
|
- league: "英超"
|
|
league_code: "EPL"
|
|
season_id: 24646
|
|
priority: 10
|
|
active: true
|
|
- league: "西甲"
|
|
league_code: "LAL"
|
|
season_id: 24651
|
|
priority: 11
|
|
active: true
|
|
- league: "德甲"
|
|
league_code: "BUN"
|
|
season_id: 24648
|
|
priority: 12
|
|
active: true
|
|
- league: "意甲"
|
|
league_code: "SA"
|
|
season_id: 24596
|
|
priority: 13
|
|
active: true
|
|
- league: "法甲"
|
|
league_code: "L1"
|
|
season_id: 24652
|
|
priority: 14
|
|
active: true
|
|
|
|
# 六合彩开奖数据源
|
|
lottery:
|
|
sources:
|
|
- name: "澳门六合彩"
|
|
code: "mc6"
|
|
category_id: 2
|
|
url: "https://4.194.133.188:8158/chajian/xam/result.txt"
|
|
- name: "香港六合彩"
|
|
code: "hk6"
|
|
category_id: 1
|
|
url: "https://4.194.133.188:8158/chajian/xg/result.txt"
|
|
|
|
# 调度器配置
|
|
scheduler:
|
|
standings_cron: "0 2 * * *" # 每天凌晨2点
|
|
schedule_cron: "0 */6 * * *" # 每6小时
|
|
live_cron: "*/5 * * * *" # 每5分钟（比赛日）
|
|
news_cron: "*/30 * * * *" # 每30分钟
|
|
content_cron: "*/10 * * * *" # 每10分钟补全文章详情
|
|
video_cron: "*/30 * * * *" # 每30分钟采集视频列表
|
|
|
|
# 日志配置
|
|
logging:
|
|
level: "INFO"
|
|
file: "logs/crawler.log"
|
|
max_size_mb: 50
|
|
backup_count: 5
|
|
format: "{time:YYYY-MM-DD HH:mm:ss} | {level:<8} | {name}:{function}:{line} - {message}"
|
|
|
|
# 告警配置
|
|
alert:
|
|
dispatch_enabled: false
|
|
wecom_webhook: ""
|
|
cooldown_seconds: 1800
|
|
request_timeout: 10
|