迁移目录
This commit is contained in:
hajimi
@@ -0,0 +1,238 @@
|
||||
"""
|
||||
懂球帝比赛实时详情采集脚本
|
||||
- 每5秒轮询 la_match 表，找出已到开赛时间但未结束的比赛
|
||||
- 异步并发采集每场比赛的 liveDetail 页面
|
||||
- 解析 __NUXT__ 数据，更新 la_match 比分/状态，写入 la_match_data
|
||||
- 每个异步任务间隔5-10秒随机
|
||||
- 比赛结束后自动停止该场采集
|
||||
"""
|
||||
import asyncio
|
||||
import random
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from curl_cffi import requests as curl_requests
|
||||
from loguru import logger
|
||||
|
||||
from src.core.config import get_config
|
||||
from src.storage.database import Database
|
||||
from src.parser.nuxt_parser import parse_nuxt_match_sample
|
||||
|
||||
LIVE_DETAIL_URL = "https://www.dongqiudi.com/liveDetail/{match_id}"
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Referer": "https://www.dongqiudi.com/",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
STATUS_MAP = {"Fixture": 0, "Playing": 1, "Played": 2}
|
||||
|
||||
# 正在采集中的 match_id 集合，防止重复调度
|
||||
_active_tasks: dict = {} # match_id -> asyncio.Task
|
||||
|
||||
# 全局停止信号
|
||||
_stop_event = asyncio.Event()
|
||||
|
||||
|
||||
def fetch_live_detail(match_id: int) -> str:
|
||||
"""同步请求 liveDetail 页面（在线程池中执行）"""
|
||||
url = LIVE_DETAIL_URL.format(match_id=match_id)
|
||||
resp = curl_requests.get(url, headers=HEADERS, impersonate="chrome", timeout=20)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
|
||||
|
||||
async def crawl_single_match(match_id: int, match_info: dict, db: Database):
|
||||
"""采集单场比赛的实时数据，循环直到比赛结束"""
|
||||
home = match_info.get("home_team", "")
|
||||
away = match_info.get("away_team", "")
|
||||
league = match_info.get("league_name", "")
|
||||
logger.info(f"[开始采集] match_id={match_id} {league} {home} vs {away}")
|
||||
|
||||
consecutive_errors = 0
|
||||
max_errors = 5
|
||||
|
||||
while not _stop_event.is_set():
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
html = await loop.run_in_executor(None, fetch_live_detail, match_id)
|
||||
|
||||
match_sample = parse_nuxt_match_sample(html)
|
||||
if not match_sample:
|
||||
logger.warning(f"[{match_id}] 解析 __NUXT__ 失败，跳过本轮")
|
||||
consecutive_errors += 1
|
||||
if consecutive_errors >= max_errors:
|
||||
logger.error(f"[{match_id}] 连续 {max_errors} 次失败，停止采集")
|
||||
break
|
||||
await asyncio.sleep(random.uniform(5, 10))
|
||||
continue
|
||||
|
||||
consecutive_errors = 0
|
||||
|
||||
# 提取关键数据
|
||||
status_str = str(match_sample.get("status", ""))
|
||||
status_int = STATUS_MAP.get(status_str, 0)
|
||||
fs_a = match_sample.get("fs_A", "")
|
||||
fs_b = match_sample.get("fs_B", "")
|
||||
home_score = int(fs_a) if fs_a and str(fs_a).isdigit() else 0
|
||||
away_score = int(fs_b) if fs_b and str(fs_b).isdigit() else 0
|
||||
hts_a = str(match_sample.get("hts_A", ""))
|
||||
hts_b = str(match_sample.get("hts_B", ""))
|
||||
half_score = f"{hts_a}-{hts_b}" if hts_a and hts_b and hts_a != "" and hts_b != "" else ""
|
||||
minute = str(match_sample.get("minute", ""))
|
||||
minute_period = str(match_sample.get("minute_period", ""))
|
||||
current_minute = minute_period if minute_period else minute
|
||||
|
||||
logger.info(
|
||||
f"[{match_id}] {league} {home} {home_score}-{away_score} {away} | "
|
||||
f"status={status_str} minute={current_minute}"
|
||||
)
|
||||
|
||||
# 更新 la_match
|
||||
await db.update_match_live(
|
||||
match_id=match_id,
|
||||
status=status_int,
|
||||
home_score=home_score,
|
||||
away_score=away_score,
|
||||
half_score=half_score,
|
||||
current_minute=current_minute,
|
||||
)
|
||||
|
||||
# 写入 la_match_data
|
||||
await db.upsert_match_data(match_sample)
|
||||
|
||||
# 比赛结束
|
||||
if status_str == "Played":
|
||||
logger.info(f"[{match_id}] 比赛已结束: {home} {home_score}-{away_score} {away}")
|
||||
break
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.info(f"[{match_id}] 任务被取消")
|
||||
break
|
||||
except Exception as e:
|
||||
consecutive_errors += 1
|
||||
logger.error(f"[{match_id}] 采集异常: {e}")
|
||||
if consecutive_errors >= max_errors:
|
||||
logger.error(f"[{match_id}] 连续 {max_errors} 次异常，停止采集")
|
||||
break
|
||||
|
||||
# 随机间隔 5-10 秒
|
||||
delay = random.uniform(5, 10)
|
||||
try:
|
||||
await asyncio.wait_for(_stop_event.wait(), timeout=delay)
|
||||
break # stop_event 被设置，退出
|
||||
except asyncio.TimeoutError:
|
||||
pass # 正常超时，继续下一轮
|
||||
|
||||
logger.info(f"[结束采集] match_id={match_id} {league} {home} vs {away}")
|
||||
|
||||
|
||||
async def poll_loop(db: Database, interval: int = 5):
|
||||
"""主轮询循环：每 interval 秒检查一次需要采集的比赛"""
|
||||
logger.info(f"启动轮询循环，间隔 {interval} 秒")
|
||||
|
||||
while not _stop_event.is_set():
|
||||
try:
|
||||
now_ts = int(time.time())
|
||||
matches = await db.get_live_matches(now_ts)
|
||||
|
||||
# 清理已完成的任务
|
||||
done_ids = [mid for mid, task in _active_tasks.items() if task.done()]
|
||||
for mid in done_ids:
|
||||
del _active_tasks[mid]
|
||||
|
||||
if matches:
|
||||
new_count = 0
|
||||
for m in matches:
|
||||
mid = int(m["match_id"])
|
||||
if mid in _active_tasks:
|
||||
continue # 已在采集中
|
||||
task = asyncio.create_task(crawl_single_match(mid, m, db))
|
||||
_active_tasks[mid] = task
|
||||
new_count += 1
|
||||
# 每启动一个任务间隔 1-2 秒，避免瞬间并发太多
|
||||
if new_count > 0:
|
||||
await asyncio.sleep(random.uniform(1, 2))
|
||||
|
||||
if new_count > 0:
|
||||
logger.info(f"新启动 {new_count} 个采集任务，当前活跃: {len(_active_tasks)}")
|
||||
else:
|
||||
active_count = len(_active_tasks)
|
||||
if active_count > 0:
|
||||
logger.debug(f"无新比赛，当前活跃任务: {active_count}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"轮询异常: {e}")
|
||||
|
||||
# 等待 interval 秒或停止信号
|
||||
try:
|
||||
await asyncio.wait_for(_stop_event.wait(), timeout=interval)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
# 停止所有活跃任务
|
||||
logger.info(f"停止所有活跃任务 ({len(_active_tasks)} 个)...")
|
||||
for mid, task in _active_tasks.items():
|
||||
task.cancel()
|
||||
if _active_tasks:
|
||||
await asyncio.gather(*_active_tasks.values(), return_exceptions=True)
|
||||
_active_tasks.clear()
|
||||
|
||||
|
||||
async def run(interval: int = 5):
|
||||
db = Database()
|
||||
try:
|
||||
await db.connect()
|
||||
logger.info("数据库连接成功")
|
||||
await poll_loop(db, interval)
|
||||
finally:
|
||||
await db.close()
|
||||
logger.info("数据库连接已关闭")
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="懂球帝比赛实时详情采集")
|
||||
parser.add_argument("--interval", type=int, default=5, help="轮询间隔秒数 (默认5)")
|
||||
args = parser.parse_args()
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("懂球帝比赛实时详情采集 (liveDetail)")
|
||||
logger.info(f"轮询间隔: {args.interval} 秒")
|
||||
logger.info("按 Ctrl+C 停止")
|
||||
logger.info("=" * 60)
|
||||
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
def handle_signal():
|
||||
logger.info("收到停止信号...")
|
||||
_stop_event.set()
|
||||
|
||||
try:
|
||||
loop.add_signal_handler(signal.SIGINT, handle_signal)
|
||||
loop.add_signal_handler(signal.SIGTERM, handle_signal)
|
||||
except NotImplementedError:
|
||||
pass # Windows 不支持 add_signal_handler
|
||||
|
||||
try:
|
||||
loop.run_until_complete(run(interval=args.interval))
|
||||
except KeyboardInterrupt:
|
||||
logger.info("收到 Ctrl+C，正在停止...")
|
||||
_stop_event.set()
|
||||
loop.run_until_complete(asyncio.sleep(1))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user