haha/sbnews
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

迁移目录

Browse Source
This commit is contained in:
hajimi
2026-06-12 22:27:18 +08:00
parent 7381097582
commit cd44cd6e47
92 changed files with 1839 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/crawler/scripts/crawl_article_detail.py
+135
View File
@@ -0,0 +1,135 @@
"""
懂球帝文章正文采集脚本
- 从数据库读取已入库但无正文的文章
- 请求文章详情页HTML，提取 div.con 正文内容
- 将正文HTML写入 la_article.content
用法:
python scripts/crawl_article_detail.py # 采集最多50篇
python scripts/crawl_article_detail.py --limit 200 # 采集最多200篇
"""
import asyncio
import random
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from bs4 import BeautifulSoup
from curl_cffi import requests as curl_requests
from loguru import logger
from src.core.config import get_config
from src.storage.database import Database
DETAIL_URL = "https://www.dongqiudi.com/articles/{article_id}"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Referer": "https://www.dongqiudi.com/",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
def fetch_article_html(article_id: int) -> str:
url = DETAIL_URL.format(article_id=article_id)
resp = curl_requests.get(
url, headers=HEADERS,
impersonate="chrome", timeout=20,
)
resp.raise_for_status()
return resp.text
def extract_content(html: str) -> str:
soup = BeautifulSoup(html, "html.parser")
con_div = soup.select_one("div.con")
if not con_div:
return ""
inner = con_div.find("div", style=lambda v: v and "display:none" in v.replace(" ", ""))
target = inner if inner else con_div
for img in target.find_all("img"):
src = img.get("data-src") or img.get("orig-src") or img.get("src", "")
if src:
img["src"] = src
for attr in ["data-src", "orig-src", "data-width", "data-height"]:
if img.has_attr(attr):
del img[attr]
return str(target.decode_contents()).strip()
async def run(limit: int = 50):
cfg = get_config()
db = Database()
try:
articles = await db.get_articles_without_content(limit=limit)
logger.info(f"待采集正文: {len(articles)} 篇")
if not articles:
logger.info("所有文章已有正文，无需采集")
return
success_count = 0
fail_count = 0
for i, art in enumerate(articles, 1):
article_id = art["article_id"]
title = art["title"][:30]
logger.info(f"[{i}/{len(articles)}] article_id={article_id} {title}...")
try:
html = fetch_article_html(article_id)
content = extract_content(html)
if not content:
logger.warning(f" 正文为空，跳过")
fail_count += 1
continue
await db.update_article_content(article_id, content)
success_count += 1
logger.info(f" 正文长度: {len(content)} 字符 ✓")
except Exception as e:
logger.error(f" 采集失败: {e}")
fail_count += 1
delay = random.uniform(1.5, 3.0)
time.sleep(delay)
logger.info(f"✅ 正文采集完成: 成功 {success_count}, 失败 {fail_count}")
await db.log_crawl(
crawl_type="article_detail",
target="dongqiudi_article_html",
success=True,
elapsed=0,
record_count=success_count,
)
except Exception as e:
logger.error(f"采集失败: {e}", exc_info=True)
raise
finally:
await db.close()
def main():
import argparse
parser = argparse.ArgumentParser(description="懂球帝文章正文采集")
parser.add_argument("--limit", type=int, default=50, help="最多采集篇数 (默认50)")
args = parser.parse_args()
logger.info("=" * 60)
logger.info("懂球帝文章正文采集")
logger.info(f"最多采集 {args.limit} 篇")
logger.info("=" * 60)
asyncio.run(run(args.limit))
if __name__ == "__main__":
main()
docker/crawler/scripts/crawl_articles.py
+180
View File
@@ -0,0 +1,180 @@
"""
懂球帝文章资讯采集脚本
- 通过 /api/app/tabs/iphone/{tab_id}.json 接口获取文章列表
- 支持多Tab采集 + 分页翻页
- 按 article_id 去重，已有数据跳过
用法:
python scripts/crawl_articles.py # 采集所有Tab第1页
python scripts/crawl_articles.py --pages 5 # 每个Tab采集5页
python scripts/crawl_articles.py --tabs 1,3,5 # 只采集指定Tab
"""
import asyncio
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from curl_cffi import requests as curl_requests
from loguru import logger
from src.core.config import get_config
from src.storage.database import Database
API_URL = "https://www.dongqiudi.com/api/app/tabs/iphone/{tab_id}.json"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Referer": "https://www.dongqiudi.com/",
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TAB_CONFIG = {
1: "头条",
3: "英超",
4: "意甲",
5: "西甲",
6: "德甲",
}
def fetch_tab_page(tab_id: int, page_url: str = "") -> dict:
if not page_url:
page_url = API_URL.format(tab_id=tab_id)
resp = curl_requests.get(
page_url, headers=HEADERS,
impersonate="chrome", timeout=20,
)
resp.raise_for_status()
return resp.json()
def collect_tab(tab_id: int, max_pages: int = 1) -> list:
all_articles = []
page_url = ""
for page_idx in range(1, max_pages + 1):
label = TAB_CONFIG.get(tab_id, f"Tab{tab_id}")
logger.info(f"[{label}] 第{page_idx}页 请求...")
try:
data = fetch_tab_page(tab_id, page_url)
except Exception as e:
logger.error(f"[{label}] 第{page_idx}页 请求失败: {e}")
break
articles = data.get("articles", [])
logger.info(f"[{label}] 第{page_idx}页 返回 {len(articles)} 条文章")
if not articles:
break
all_articles.extend(articles)
next_url = data.get("next", "")
if not next_url:
logger.info(f"[{label}] 无下一页，停止")
break
page_url = next_url
time.sleep(2)
return all_articles
async def run(tab_ids: list, max_pages: int = 1):
cfg = get_config()
db = Database()
try:
existing_ids = await db.get_existing_article_ids()
cate_map = await db.get_article_cate_map()
logger.info(f"数据库已有 {len(existing_ids)} 条文章, {len(cate_map)} 个分类")
total_new = 0
total_skip = 0
for tab_id in tab_ids:
label = TAB_CONFIG.get(tab_id, f"Tab{tab_id}")
cid = cate_map.get(label, 0)
if not cid:
logger.warning(f"[{label}] 未找到对应分类，使用 cid=0")
all_articles = collect_tab(tab_id, max_pages)
unique = {}
for a in all_articles:
aid = int(a.get("id", 0))
if aid and aid not in unique:
unique[aid] = a
new_items = [a for aid, a in unique.items() if aid not in existing_ids]
skip_count = len(unique) - len(new_items)
logger.info(f"[{label}] 去重: {len(unique)} 条唯一, 新增: {len(new_items)}, 跳过: {skip_count}")
if new_items:
count = await db.upsert_articles(new_items, cid=cid)
total_new += count
existing_ids.update(int(a["id"]) for a in new_items)
total_skip += skip_count
time.sleep(1)
logger.info(f"✅ 采集完成: 新增 {total_new} 条, 跳过 {total_skip} 条")
await db.log_crawl(
crawl_type="articles",
target="dongqiudi_tabs_api",
success=True,
elapsed=0,
record_count=total_new,
)
except Exception as e:
logger.error(f"采集失败: {e}", exc_info=True)
try:
await db.log_crawl(
crawl_type="articles",
target="dongqiudi_tabs_api",
success=False,
elapsed=0,
error=str(e),
)
except Exception:
pass
raise
finally:
await db.close()
def main():
import argparse
parser = argparse.ArgumentParser(
description="懂球帝文章资讯采集",
epilog="示例:\n"
" python crawl_articles.py # 所有Tab各1页\n"
" python crawl_articles.py --pages 5 # 每Tab采5页\n"
" python crawl_articles.py --tabs 1,3,5 # 指定Tab\n",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--pages", type=int, default=1, help="每个Tab采集页数 (默认1)")
parser.add_argument("--tabs", type=str, default="", help="Tab ID列表,逗号分隔 (默认全部)")
args = parser.parse_args()
if args.tabs:
tab_ids = [int(t.strip()) for t in args.tabs.split(",") if t.strip()]
else:
tab_ids = list(TAB_CONFIG.keys())
tab_names = [TAB_CONFIG.get(t, f"Tab{t}") for t in tab_ids]
logger.info("=" * 60)
logger.info("懂球帝文章资讯采集")
logger.info(f"Tabs: {', '.join(tab_names)}, 每Tab {args.pages} 页")
logger.info("=" * 60)
asyncio.run(run(tab_ids, args.pages))
if __name__ == "__main__":
main()
docker/crawler/scripts/crawl_live_detail.py
+238
View File
@@ -0,0 +1,238 @@
"""
懂球帝比赛实时详情采集脚本
- 每5秒轮询 la_match 表，找出已到开赛时间但未结束的比赛
- 异步并发采集每场比赛的 liveDetail 页面
- 解析 __NUXT__ 数据，更新 la_match 比分/状态，写入 la_match_data
- 每个异步任务间隔5-10秒随机
- 比赛结束后自动停止该场采集
"""
import asyncio
import random
import signal
import sys
import time
from datetime import datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from curl_cffi import requests as curl_requests
from loguru import logger
from src.core.config import get_config
from src.storage.database import Database
from src.parser.nuxt_parser import parse_nuxt_match_sample
LIVE_DETAIL_URL = "https://www.dongqiudi.com/liveDetail/{match_id}"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Referer": "https://www.dongqiudi.com/",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
STATUS_MAP = {"Fixture": 0, "Playing": 1, "Played": 2}
# 正在采集中的 match_id 集合，防止重复调度
_active_tasks: dict = {} # match_id -> asyncio.Task
# 全局停止信号
_stop_event = asyncio.Event()
def fetch_live_detail(match_id: int) -> str:
"""同步请求 liveDetail 页面（在线程池中执行）"""
url = LIVE_DETAIL_URL.format(match_id=match_id)
resp = curl_requests.get(url, headers=HEADERS, impersonate="chrome", timeout=20)
resp.raise_for_status()
return resp.text
async def crawl_single_match(match_id: int, match_info: dict, db: Database):
"""采集单场比赛的实时数据，循环直到比赛结束"""
home = match_info.get("home_team", "")
away = match_info.get("away_team", "")
league = match_info.get("league_name", "")
logger.info(f"[开始采集] match_id={match_id} {league} {home} vs {away}")
consecutive_errors = 0
max_errors = 5
while not _stop_event.is_set():
try:
loop = asyncio.get_event_loop()
html = await loop.run_in_executor(None, fetch_live_detail, match_id)
match_sample = parse_nuxt_match_sample(html)
if not match_sample:
logger.warning(f"[{match_id}] 解析 __NUXT__ 失败，跳过本轮")
consecutive_errors += 1
if consecutive_errors >= max_errors:
logger.error(f"[{match_id}] 连续 {max_errors} 次失败，停止采集")
break
await asyncio.sleep(random.uniform(5, 10))
continue
consecutive_errors = 0
# 提取关键数据
status_str = str(match_sample.get("status", ""))
status_int = STATUS_MAP.get(status_str, 0)
fs_a = match_sample.get("fs_A", "")
fs_b = match_sample.get("fs_B", "")
home_score = int(fs_a) if fs_a and str(fs_a).isdigit() else 0
away_score = int(fs_b) if fs_b and str(fs_b).isdigit() else 0
hts_a = str(match_sample.get("hts_A", ""))
hts_b = str(match_sample.get("hts_B", ""))
half_score = f"{hts_a}-{hts_b}" if hts_a and hts_b and hts_a != "" and hts_b != "" else ""
minute = str(match_sample.get("minute", ""))
minute_period = str(match_sample.get("minute_period", ""))
current_minute = minute_period if minute_period else minute
logger.info(
f"[{match_id}] {league} {home} {home_score}-{away_score} {away} | "
f"status={status_str} minute={current_minute}"
)
# 更新 la_match
await db.update_match_live(
match_id=match_id,
status=status_int,
home_score=home_score,
away_score=away_score,
half_score=half_score,
current_minute=current_minute,
)
# 写入 la_match_data
await db.upsert_match_data(match_sample)
# 比赛结束
if status_str == "Played":
logger.info(f"[{match_id}] 比赛已结束: {home} {home_score}-{away_score} {away}")
break
except asyncio.CancelledError:
logger.info(f"[{match_id}] 任务被取消")
break
except Exception as e:
consecutive_errors += 1
logger.error(f"[{match_id}] 采集异常: {e}")
if consecutive_errors >= max_errors:
logger.error(f"[{match_id}] 连续 {max_errors} 次异常，停止采集")
break
# 随机间隔 5-10 秒
delay = random.uniform(5, 10)
try:
await asyncio.wait_for(_stop_event.wait(), timeout=delay)
break # stop_event 被设置，退出
except asyncio.TimeoutError:
pass # 正常超时，继续下一轮
logger.info(f"[结束采集] match_id={match_id} {league} {home} vs {away}")
async def poll_loop(db: Database, interval: int = 5):
"""主轮询循环：每 interval 秒检查一次需要采集的比赛"""
logger.info(f"启动轮询循环，间隔 {interval} 秒")
while not _stop_event.is_set():
try:
now_ts = int(time.time())
matches = await db.get_live_matches(now_ts)
# 清理已完成的任务
done_ids = [mid for mid, task in _active_tasks.items() if task.done()]
for mid in done_ids:
del _active_tasks[mid]
if matches:
new_count = 0
for m in matches:
mid = int(m["match_id"])
if mid in _active_tasks:
continue # 已在采集中
task = asyncio.create_task(crawl_single_match(mid, m, db))
_active_tasks[mid] = task
new_count += 1
# 每启动一个任务间隔 1-2 秒，避免瞬间并发太多
if new_count > 0:
await asyncio.sleep(random.uniform(1, 2))
if new_count > 0:
logger.info(f"新启动 {new_count} 个采集任务，当前活跃: {len(_active_tasks)}")
else:
active_count = len(_active_tasks)
if active_count > 0:
logger.debug(f"无新比赛，当前活跃任务: {active_count}")
except Exception as e:
logger.error(f"轮询异常: {e}")
# 等待 interval 秒或停止信号
try:
await asyncio.wait_for(_stop_event.wait(), timeout=interval)
break
except asyncio.TimeoutError:
pass
# 停止所有活跃任务
logger.info(f"停止所有活跃任务 ({len(_active_tasks)} 个)...")
for mid, task in _active_tasks.items():
task.cancel()
if _active_tasks:
await asyncio.gather(*_active_tasks.values(), return_exceptions=True)
_active_tasks.clear()
async def run(interval: int = 5):
db = Database()
try:
await db.connect()
logger.info("数据库连接成功")
await poll_loop(db, interval)
finally:
await db.close()
logger.info("数据库连接已关闭")
def main():
import argparse
parser = argparse.ArgumentParser(description="懂球帝比赛实时详情采集")
parser.add_argument("--interval", type=int, default=5, help="轮询间隔秒数 (默认5)")
args = parser.parse_args()
logger.info("=" * 60)
logger.info("懂球帝比赛实时详情采集 (liveDetail)")
logger.info(f"轮询间隔: {args.interval} 秒")
logger.info("按 Ctrl+C 停止")
logger.info("=" * 60)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
def handle_signal():
logger.info("收到停止信号...")
_stop_event.set()
try:
loop.add_signal_handler(signal.SIGINT, handle_signal)
loop.add_signal_handler(signal.SIGTERM, handle_signal)
except NotImplementedError:
pass # Windows 不支持 add_signal_handler
try:
loop.run_until_complete(run(interval=args.interval))
except KeyboardInterrupt:
logger.info("收到 Ctrl+C，正在停止...")
_stop_event.set()
loop.run_until_complete(asyncio.sleep(1))
finally:
loop.close()
if __name__ == "__main__":
main()
docker/crawler/scripts/crawl_matches.py
+187
View File
@@ -0,0 +1,187 @@
"""
懂球帝比赛数据采集脚本
- 通过 /api/data/tab/new/important 接口获取比赛记录
- 按周循环翻页 (start + 7天)
- 按天匹配，已有数据跳过
- 每天执行一次
"""
import asyncio
import sys
import time
from datetime import datetime, timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from curl_cffi import requests as curl_requests
from loguru import logger
from src.core.config import get_config
from src.storage.database import Database
API_URL = "https://www.dongqiudi.com/api/data/tab/new/important"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Referer": "https://www.dongqiudi.com/",
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
def fetch_page(start_str: str) -> dict:
params = {
"start": f"{start_str}next",
"init": "1",
"platform": "www",
}
resp = curl_requests.get(
API_URL, params=params, headers=HEADERS,
impersonate="chrome", timeout=20,
)
resp.raise_for_status()
return resp.json()
def collect_range(start_date: datetime, end_date: datetime) -> list:
all_items = []
current = start_date
page = 0
while current < end_date:
page += 1
start_str = current.strftime("%Y-%m-%d") + " 00:00:00"
logger.info(f"[第{page}页] 请求: start={start_str}")
try:
data = fetch_page(start_str)
except Exception as e:
logger.error(f"请求失败: {e}")
current += timedelta(days=7)
time.sleep(2)
continue
items = data.get("list", [])
logger.info(f" 返回 {len(items)} 条记录")
for item in items:
all_items.append(item)
current += timedelta(days=7)
time.sleep(3)
return all_items
async def run(start_date: datetime, end_date: datetime):
cfg = get_config()
db = Database()
try:
existing_ids = await db.get_existing_match_ids()
logger.info(f"数据库已有 {len(existing_ids)} 条比赛记录(有match_id)")
days = (end_date - start_date).days
logger.info(f"采集范围: {start_date.strftime('%Y-%m-%d')} ~ {end_date.strftime('%Y-%m-%d')} ({days}天)")
all_items = collect_range(start_date, end_date)
unique_items = {}
for item in all_items:
mid = int(item["match_id"])
if mid not in unique_items:
unique_items[mid] = item
logger.info(f"去重后共 {len(unique_items)} 条唯一比赛")
new_items = [item for mid, item in unique_items.items() if mid not in existing_ids]
logger.info(f"新增比赛: {new_items.__len__()} 条 (跳过已有 {len(unique_items) - len(new_items)} 条)")
if not new_items:
logger.info("没有新比赛需要入库")
return
fixture_items = [i for i in new_items if i.get("status") == "Fixture"]
played_items = [i for i in new_items if i.get("status") == "Played"]
logger.info(f" 未开始: {len(fixture_items)} 条, 已结束: {len(played_items)} 条")
count = await db.upsert_la_match(new_items)
logger.info(f"✅ 入库完成: {count} 条比赛记录")
await db.log_crawl(
crawl_type="match_important",
target=API_URL,
success=True,
elapsed=0,
record_count=count,
)
except Exception as e:
logger.error(f"采集失败: {e}", exc_info=True)
try:
await db.log_crawl(
crawl_type="match_important",
target=API_URL,
success=False,
elapsed=0,
error=str(e),
)
except Exception:
pass
raise
finally:
await db.close()
def parse_date(s: str) -> datetime:
for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y%m%d"):
try:
return datetime.strptime(s, fmt)
except ValueError:
continue
raise ValueError(f"无法解析日期: {s}，支持格式: YYYY-MM-DD / YYYY/MM/DD / YYYYMMDD")
def main():
import argparse
parser = argparse.ArgumentParser(
description="懂球帝比赛数据采集",
epilog="示例:\n"
" python crawl_matches.py --weeks 4 # 从今天起采集4周\n"
" python crawl_matches.py --start 2026-01-01 --end 2026-02-01 # 指定时间段\n"
" python crawl_matches.py --start 2026-01-01 # 从指定日期到今天\n",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--weeks", type=int, default=None, help="从今天起采集未来N周 (默认4)")
parser.add_argument("--start", type=str, default=None, help="起始日期 (YYYY-MM-DD)")
parser.add_argument("--end", type=str, default=None, help="结束日期 (YYYY-MM-DD)，不指定则到今天")
args = parser.parse_args()
today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
if args.start:
start_date = parse_date(args.start)
if args.end:
end_date = parse_date(args.end)
else:
end_date = today + timedelta(days=1)
else:
weeks = args.weeks or 4
start_date = today
end_date = today + timedelta(weeks=weeks)
if start_date >= end_date:
logger.error(f"起始日期 {start_date.strftime('%Y-%m-%d')} >= 结束日期 {end_date.strftime('%Y-%m-%d')}")
sys.exit(1)
logger.info("=" * 60)
logger.info("懂球帝比赛数据采集 (important)")
logger.info(f"{start_date.strftime('%Y-%m-%d')} ~ {end_date.strftime('%Y-%m-%d')}")
logger.info("=" * 60)
asyncio.run(run(start_date, end_date))
if __name__ == "__main__":
main()
docker/crawler/scripts/docker_task_runner.py
+439
View File
@@ -0,0 +1,439 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import os
import shlex
import socket
import subprocess
import sys
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Iterable, Optional
import pymysql
import yaml
APP_DIR = Path(__file__).resolve().parents[1]
if str(APP_DIR) not in sys.path:
sys.path.insert(0, str(APP_DIR))
from main import get_action_timeout # noqa: E402
from src.core.config import load_config # noqa: E402
STATUS_RUNNING = 0
STATUS_SUCCESS = 1
STATUS_FAILED = 2
STATUS_SKIPPED = 3
OUTPUT_LIMIT = 5000
@dataclass
class CrawlerTask:
task_key: str
task_name: str
action: str
cron_expression: str
active: bool = True
timeout: Optional[int] = None
@classmethod
def from_dict(cls, row: dict) -> "CrawlerTask":
action = str(row.get("action") or row.get("params") or "").strip()
return cls(
task_key=str(row.get("key") or row.get("task_key") or action).strip(),
task_name=str(row.get("name") or row.get("task_name") or action).strip(),
action=action,
cron_expression=str(row.get("cron") or row.get("cron_expression") or "* * * * *").strip(),
active=bool(row.get("active", True)),
timeout=int(row["timeout"]) if row.get("timeout") not in (None, "") else None,
)
def resolved_timeout(self) -> int:
return int(self.timeout or get_action_timeout(self.action))
@dataclass
class CommandResult:
exit_code: int
output: str
elapsed: float
timed_out: bool = False
@dataclass
class TaskRunResult:
log_id: int
status: int
output: str
elapsed: float
error_message: str = ""
class TaskLock:
def __init__(self, action: str, lock_dir: Path):
safe_action = "".join(ch if ch.isalnum() or ch in ("-", "_") else "_" for ch in str(action or "all"))
self.lock_dir = Path(lock_dir)
self.path = self.lock_dir / f"{safe_action}.lock"
self._fp = None
self._locked = False
def acquire(self) -> bool:
self.lock_dir.mkdir(parents=True, exist_ok=True)
self._fp = open(self.path, "a+b")
try:
self._fp.seek(0)
if os.name == "nt":
import msvcrt
msvcrt.locking(self._fp.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
self._locked = True
self._fp.seek(0)
self._fp.truncate()
self._fp.write(f"pid={os.getpid()} started_at={int(time.time())}\n".encode("utf-8"))
self._fp.flush()
return True
except (BlockingIOError, OSError):
self.release()
return False
def release(self) -> None:
if not self._fp:
return
try:
if self._locked:
if os.name == "nt":
import msvcrt
self._fp.seek(0)
msvcrt.locking(self._fp.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN)
finally:
self._locked = False
self._fp.close()
self._fp = None
class TaskLogStore:
def __init__(self):
cfg = load_config().database
self.prefix = cfg.prefix
self.conn = pymysql.connect(
host=cfg.host,
port=cfg.port,
user=cfg.username,
password=cfg.password,
database=cfg.database,
charset=cfg.charset,
autocommit=True,
)
self.ensure_table()
@property
def table(self) -> str:
return f"`{self.prefix}crawler_task_log`"
def close(self) -> None:
self.conn.close()
def ensure_table(self) -> None:
sql = f"""
CREATE TABLE IF NOT EXISTS {self.table} (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`task_key` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '任务标识',
`task_name` VARCHAR(100) NOT NULL DEFAULT '' COMMENT '任务名称',
`action` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '爬虫动作',
`cron_expression` VARCHAR(64) NOT NULL DEFAULT '' COMMENT 'cron规则',
`source` VARCHAR(32) NOT NULL DEFAULT 'docker' COMMENT '来源',
`container_id` VARCHAR(128) NOT NULL DEFAULT '' COMMENT '容器ID',
`status` TINYINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '0执行中 1成功 2失败 3跳过',
`trigger_type` TINYINT UNSIGNED NOT NULL DEFAULT 1 COMMENT '1自动 2手动',
`output` TEXT NULL COMMENT '执行输出',
`error_message` TEXT NULL COMMENT '错误信息',
`elapsed` DECIMAL(10,2) NOT NULL DEFAULT 0.00 COMMENT '耗时秒',
`started_at` INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '开始时间',
`finished_at` INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '结束时间',
`create_time` INT UNSIGNED NOT NULL DEFAULT 0,
`update_time` INT UNSIGNED NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
KEY `idx_action_status` (`action`, `status`),
KEY `idx_create_time` (`create_time`),
KEY `idx_task_key` (`task_key`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Docker爬虫任务执行日志'
"""
with self.conn.cursor() as cur:
cur.execute(sql)
def create_running(self, task: CrawlerTask) -> int:
now = int(time.time())
with self.conn.cursor() as cur:
cur.execute(
f"""
INSERT INTO {self.table}
(`task_key`, `task_name`, `action`, `cron_expression`, `source`, `container_id`,
`status`, `trigger_type`, `output`, `error_message`, `elapsed`,
`started_at`, `finished_at`, `create_time`, `update_time`)
VALUES (%s, %s, %s, %s, 'docker', %s, %s, 1, '', '', 0, %s, 0, %s, %s)
""",
(
task.task_key,
task.task_name,
task.action,
task.cron_expression,
socket.gethostname(),
STATUS_RUNNING,
now,
now,
now,
),
)
return int(cur.lastrowid)
def update_progress(self, log_id: int, output: str, elapsed: float) -> None:
with self.conn.cursor() as cur:
cur.execute(
f"UPDATE {self.table} SET output=%s, elapsed=%s, update_time=%s WHERE id=%s",
(output[-OUTPUT_LIMIT:], round(elapsed, 2), int(time.time()), log_id),
)
def finish(self, log_id: int, status: int, output: str, elapsed: float, error_message: str = "") -> None:
now = int(time.time())
with self.conn.cursor() as cur:
cur.execute(
f"""
UPDATE {self.table}
SET status=%s, output=%s, error_message=%s, elapsed=%s, finished_at=%s, update_time=%s
WHERE id=%s
""",
(status, output[-OUTPUT_LIMIT:], error_message[:2000], round(elapsed, 2), now, now, log_id),
)
def load_tasks(path: Path) -> list[CrawlerTask]:
raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {}
rows = raw if isinstance(raw, list) else raw.get("tasks", [])
return [CrawlerTask.from_dict(row) for row in rows]
def render_crontab(tasks: Iterable[CrawlerTask], python_bin: str = "/usr/local/bin/python", app_dir: str = "/app") -> str:
lines = [
"SHELL=/bin/bash",
"BASH_ENV=/etc/sport-era-crawler/env.sh",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"",
]
for task in tasks:
if not task.active:
continue
args = [
python_bin,
"scripts/docker_task_runner.py",
"run",
task.action,
"--task-key",
task.task_key,
"--task-name",
task.task_name,
"--cron-expression",
task.cron_expression,
]
if task.timeout:
args.extend(["--timeout", str(task.timeout)])
cmd = " ".join(shlex.quote(str(arg)) for arg in args)
lines.append(f"{task.cron_expression} root cd {shlex.quote(app_dir)} && {cmd}")
lines.append("")
return "\n".join(lines)
def run_main_action(action: str, timeout: int, output_callback: Callable[[str], None]) -> CommandResult:
start = time.monotonic()
env = os.environ.copy()
env["PYTHONUNBUFFERED"] = "1"
proc = subprocess.Popen(
[sys.executable, "main.py", action],
cwd=str(APP_DIR),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding="utf-8",
errors="replace",
env=env,
)
output_parts: list[str] = []
timed_out = False
if os.name == "nt":
try:
stdout, _ = proc.communicate(timeout=timeout)
stdout = stdout or ""
output_parts.append(stdout)
if stdout:
output_callback(stdout)
except subprocess.TimeoutExpired:
timed_out = True
proc.kill()
stdout, _ = proc.communicate()
stdout = stdout or ""
output_parts.append(stdout)
if stdout:
output_callback(stdout)
else:
import selectors
assert proc.stdout is not None
selector = selectors.DefaultSelector()
selector.register(proc.stdout, selectors.EVENT_READ)
try:
while proc.poll() is None:
remaining = timeout - (time.monotonic() - start)
if remaining <= 0:
timed_out = True
proc.kill()
break
for key, _ in selector.select(timeout=min(1.0, remaining)):
line = key.fileobj.readline()
if line:
output_parts.append(line)
output_callback(line)
rest = proc.stdout.read()
if rest:
output_parts.append(rest)
output_callback(rest)
finally:
selector.close()
elapsed = round(time.monotonic() - start, 2)
return CommandResult(
exit_code=proc.returncode if proc.returncode is not None else -1,
output="".join(output_parts),
elapsed=elapsed,
timed_out=timed_out,
)
def run_task(
task: CrawlerTask,
store=None,
lock_dir: Optional[Path] = None,
command_runner: Callable[[str, int, Callable[[str], None]], CommandResult] = run_main_action,
flush_interval: float = 5.0,
) -> TaskRunResult:
if store is None:
store = TaskLogStore()
if hasattr(store, "ensure_table"):
store.ensure_table()
log_id = int(store.create_running(task))
start = time.monotonic()
lock = TaskLock(task.action, Path(lock_dir or os.environ.get("DQD_LOCK_DIR") or APP_DIR / "data" / "locks"))
output_parts: list[str] = []
last_flush = 0.0
def append_output(text: str) -> None:
nonlocal last_flush
output_parts.append(text)
now = time.monotonic()
elapsed = round(now - start, 2)
if now - last_flush >= flush_interval:
store.update_progress(log_id, "".join(output_parts), elapsed)
last_flush = now
if not lock.acquire():
output = f"{task.action} 已有任务执行中，跳过本次"
store.finish(log_id, STATUS_SKIPPED, output, 0, "")
return TaskRunResult(log_id, STATUS_SKIPPED, output, 0, "")
try:
result = command_runner(task.action, task.resolved_timeout(), append_output)
output = result.output or "".join(output_parts) or "执行完成"
if result.timed_out:
status = STATUS_FAILED
error = f"{task.action} 执行超时，超过 {task.resolved_timeout()} 秒"
elif result.exit_code == 0:
status = STATUS_SUCCESS
error = ""
else:
status = STATUS_FAILED
error = f"{task.action} 执行失败，退出码 {result.exit_code}"
store.finish(log_id, status, output, result.elapsed, error)
return TaskRunResult(log_id, status, output, result.elapsed, error)
except Exception as exc:
elapsed = round(time.monotonic() - start, 2)
output = "".join(output_parts) or str(exc)
store.finish(log_id, STATUS_FAILED, output, elapsed, str(exc))
return TaskRunResult(log_id, STATUS_FAILED, output, elapsed, str(exc))
finally:
lock.release()
def build_task_from_args(args) -> CrawlerTask:
if args.tasks:
for task in load_tasks(Path(args.tasks)):
if task.action == args.action:
if args.timeout:
task.timeout = args.timeout
return task
return CrawlerTask(
task_key=args.task_key or args.action,
task_name=args.task_name or args.action,
action=args.action,
cron_expression=args.cron_expression or "",
active=True,
timeout=args.timeout,
)
def parse_args(argv: Optional[list[str]] = None):
parser = argparse.ArgumentParser(description="Sport Era Docker crawler task runner")
sub = parser.add_subparsers(dest="command", required=True)
run = sub.add_parser("run")
run.add_argument("action")
run.add_argument("--task-key", default="")
run.add_argument("--task-name", default="")
run.add_argument("--cron-expression", default="")
run.add_argument("--timeout", type=int, default=None)
run.add_argument("--tasks", default="")
render = sub.add_parser("render-cron")
render.add_argument("--tasks", required=True)
render.add_argument("--output", required=True)
render.add_argument("--python-bin", default="/usr/local/bin/python")
render.add_argument("--app-dir", default="/app")
return parser.parse_args(argv)
def main(argv: Optional[list[str]] = None) -> int:
args = parse_args(argv)
if args.command == "render-cron":
tasks = load_tasks(Path(args.tasks))
content = render_crontab(tasks, python_bin=args.python_bin, app_dir=args.app_dir)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(content, encoding="utf-8")
output.chmod(0o644)
return 0
if args.command == "run":
store = TaskLogStore()
try:
result = run_task(build_task_from_args(args), store=store)
return 0 if result.status in (STATUS_SUCCESS, STATUS_SKIPPED) else 1
finally:
store.close()
return 1
if __name__ == "__main__":
raise SystemExit(main())
docker/crawler/scripts/fetch_match_detail.py
+159
View File
@@ -0,0 +1,159 @@
"""
获取单场比赛详情
用法: python scripts/fetch_match_detail.py <match_id> [--save]
--save 将数据写入 la_match_data 并更新 la_match
"""
import asyncio
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from curl_cffi import requests as curl_requests
from loguru import logger
from src.parser.nuxt_parser import parse_nuxt_match_sample
from src.storage.database import Database
LIVE_DETAIL_URL = "https://www.dongqiudi.com/liveDetail/{match_id}"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Referer": "https://www.dongqiudi.com/",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
STATUS_MAP = {"Fixture": 0, "Playing": 1, "Played": 2}
def fetch_html(match_id: int) -> str:
url = LIVE_DETAIL_URL.format(match_id=match_id)
resp = curl_requests.get(url, headers=HEADERS, impersonate="chrome", timeout=20)
resp.raise_for_status()
return resp.text
def display(ms: dict):
"""格式化输出比赛详情"""
status = ms.get("status", "")
fs_a = ms.get("fs_A", "")
fs_b = ms.get("fs_B", "")
hts_a = ms.get("hts_A", "")
hts_b = ms.get("hts_B", "")
print()
print("=" * 60)
print(f" 比赛ID: {ms.get('match_id')}")
print(f" 赛事: {ms.get('competition_name', '')} {ms.get('match_title', '')}")
print(f" 轮次: {ms.get('gameweek', '') or ms.get('round_name', '') or '-'}")
print("-" * 60)
print(f" 主队: {ms.get('team_A_name', '')} (ID: {ms.get('team_A_id', '')})")
print(f" 客队: {ms.get('team_B_name', '')} (ID: {ms.get('team_B_id', '')})")
print("-" * 60)
if status == "Played":
print(f" 比分: {fs_a} - {fs_b} (全场)")
if hts_a and hts_b:
print(f" 半场: {hts_a} - {hts_b}")
print(f" 状态: 已结束")
elif status == "Playing":
minute = ms.get("minute", "")
period = ms.get("minute_period", "")
print(f" 比分: {fs_a or 0} - {fs_b or 0} (进行中)")
print(f" 进行: {period} {minute}")
else:
print(f" 状态: 未开始")
print(f" 开赛时间: {ms.get('start_play', '')}")
print(f" 类型: {ms.get('cmp_type', '')}")
# 赔率
home_odds = ms.get("home", "")
draw_odds = ms.get("draw", "")
away_odds = ms.get("away", "")
if home_odds or draw_odds or away_odds:
print(f" 赔率: 主{home_odds} 平{draw_odds} 客{away_odds}")
# 直播源
tv = ms.get("livingTv", "")
if tv:
print(f" 直播: {tv}")
print("=" * 60)
# 额外信息
ht_info = ms.get("ht_info", "")
score_info = ms.get("score_info", "")
if ht_info:
print(f" 半场信息: {ht_info}")
if score_info:
print(f" 比分信息: {score_info}")
# Logo URLs
print(f"\n 赛事Logo: {ms.get('competition_bk_logo', '')}")
print(f" 主队Logo: {ms.get('team_A_logo', '')}")
print(f" 客队Logo: {ms.get('team_B_logo', '')}")
print()
async def save_to_db(ms: dict):
"""将数据写入数据库"""
db = Database()
try:
match_id = int(ms.get("match_id", 0))
status_str = str(ms.get("status", ""))
status_int = STATUS_MAP.get(status_str, 0)
fs_a = ms.get("fs_A", "")
fs_b = ms.get("fs_B", "")
home_score = int(fs_a) if fs_a and str(fs_a).isdigit() else 0
away_score = int(fs_b) if fs_b and str(fs_b).isdigit() else 0
hts_a = str(ms.get("hts_A", ""))
hts_b = str(ms.get("hts_B", ""))
half_score = f"{hts_a}-{hts_b}" if hts_a and hts_b else ""
minute_period = str(ms.get("minute_period", ""))
await db.update_match_live(
match_id=match_id,
status=status_int,
home_score=home_score,
away_score=away_score,
half_score=half_score,
current_minute=minute_period,
)
await db.upsert_match_data(ms)
logger.info(f"已保存到数据库: match_id={match_id}")
finally:
await db.close()
def main():
import argparse
parser = argparse.ArgumentParser(description="获取单场比赛详情")
parser.add_argument("match_id", type=int, help="懂球帝 match_id")
parser.add_argument("--save", action="store_true", help="写入数据库")
parser.add_argument("--json", action="store_true", help="输出原始 JSON")
args = parser.parse_args()
logger.info(f"获取比赛详情: match_id={args.match_id}")
html = fetch_html(args.match_id)
ms = parse_nuxt_match_sample(html)
if not ms:
logger.error("解析失败，无法提取比赛数据")
sys.exit(1)
if args.json:
print(json.dumps(ms, ensure_ascii=False, indent=2, default=str))
else:
display(ms)
if args.save:
asyncio.run(save_to_db(ms))
if __name__ == "__main__":
main()
docker/crawler/scripts/kb_worker.py
+708
View File
@@ -0,0 +1,708 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import asyncio
import hashlib
import html
import json
import math
import os
import re
import socket
import sys
import time
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
import aiohttp
import aiomysql
import redis.asyncio as aioredis
import redis.exceptions as redis_exceptions
from loguru import logger
APP_DIR = Path(__file__).resolve().parents[1]
if str(APP_DIR) not in sys.path:
sys.path.insert(0, str(APP_DIR))
from src.core.config import get_config, load_config # noqa: E402
STREAM_KEY = "ai:kb:sync:stream"
DEAD_STREAM_KEY = "ai:kb:sync:dead"
GROUP_NAME = "ai_kb_workers"
COLLECTION_NAME = "global"
DEFAULT_CHUNK_SIZE = 600
DEFAULT_CHUNK_OVERLAP = 80
DEFAULT_EMBEDDING_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode"
DEFAULT_EMBEDDING_MODEL = "text-embedding-v4"
DEFAULT_EMBEDDING_DIM = 1024
def now_ts() -> int:
return int(time.time())
def normalize_plain_text(text: Any) -> str:
value = html.unescape(str(text or ""))
value = re.sub(r"\s+", " ", value.strip(), flags=re.UNICODE)
return value.strip()
def normalize_html_text(text: Any) -> str:
value = str(text or "")
value = re.sub(r"(?i)<br\s*/?>|</p>|</div>|</li>", "\n", value)
value = re.sub(r"<[^>]+>", "", value)
return normalize_plain_text(value)
def truncate_text(text: str, length: int) -> str:
text = text.strip()
if len(text) <= length:
return text
return text[:length] + "..."
def split_text(text: str, chunk_size: int = DEFAULT_CHUNK_SIZE, overlap: int = DEFAULT_CHUNK_OVERLAP) -> List[str]:
text = text.strip()
if not text:
return []
if len(text) <= chunk_size:
return [text]
chunks: List[str] = []
start = 0
step = max(1, chunk_size - overlap)
while start < len(text):
chunks.append(text[start : start + chunk_size].strip())
if start + chunk_size >= len(text):
break
start += step
return [chunk for chunk in chunks if chunk]
def vector_norm(vector: Iterable[float]) -> float:
total = 0.0
for value in vector:
total += float(value) * float(value)
return round(math.sqrt(total), 8) if total > 0 else 0.0
def json_loads_maybe(value: Any, default: Any) -> Any:
if value is None:
return default
if isinstance(value, (dict, list)):
return value
try:
return json.loads(str(value))
except Exception:
return default
def json_text(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
class KbDatabase:
def __init__(self):
cfg = get_config().database
self.cfg = cfg
self.prefix = cfg.prefix
self.pool: Optional[aiomysql.Pool] = None
async def connect(self) -> None:
if self.pool:
return
self.pool = await aiomysql.create_pool(
host=self.cfg.host,
port=self.cfg.port,
user=self.cfg.username,
password=self.cfg.password,
db=self.cfg.database,
charset=self.cfg.charset,
autocommit=False,
minsize=1,
maxsize=max(2, min(10, self.cfg.pool_size)),
)
logger.info("KB worker MySQL pool ready: {}:{}/{}", self.cfg.host, self.cfg.port, self.cfg.database)
async def close(self) -> None:
if self.pool:
self.pool.close()
await self.pool.wait_closed()
self.pool = None
def table(self, name: str) -> str:
return f"`{self.prefix}{name}`"
async def fetchone(self, sql: str, args: tuple = ()) -> Optional[Dict[str, Any]]:
await self.connect()
assert self.pool is not None
async with self.pool.acquire() as conn:
async with conn.cursor(aiomysql.DictCursor) as cur:
await cur.execute(sql, args)
return await cur.fetchone()
async def fetchall(self, sql: str, args: tuple = ()) -> List[Dict[str, Any]]:
await self.connect()
assert self.pool is not None
async with self.pool.acquire() as conn:
async with conn.cursor(aiomysql.DictCursor) as cur:
await cur.execute(sql, args)
return list(await cur.fetchall())
async def build_payload(self, domain: str, subtype: str, source_id: int) -> Optional[Dict[str, Any]]:
key = f"{domain}:{subtype}"
if key == "article:article":
return await self.build_article_payload(source_id)
if key == "post:post":
return await self.build_post_payload(source_id)
if key == "lottery:draw_result":
return await self.build_lottery_draw_result_payload(source_id)
if key == "lottery:ai_history":
return await self.build_lottery_ai_history_payload(source_id)
if key == "match:event":
return await self.build_match_event_payload(source_id)
return None
async def build_article_payload(self, source_id: int) -> Optional[Dict[str, Any]]:
row = await self.fetchone(
f"SELECT * FROM {self.table('article')} WHERE id=%s AND is_show=1 AND delete_time IS NULL LIMIT 1",
(source_id,),
)
if not row:
return None
cate = await self.fetchone(f"SELECT name FROM {self.table('article_cate')} WHERE id=%s LIMIT 1", (row.get("cid") or 0,))
cate_name = str((cate or {}).get("name") or "")
ext = json_loads_maybe(row.get("ext"), {})
translated = str(ext.get("translated_content") or "").strip() if isinstance(ext, dict) else ""
content = normalize_html_text(translated or row.get("content") or "")
summary = normalize_plain_text(row.get("abstract") or row.get("desc") or "")
keywords = list(dict.fromkeys(filter(None, [
str(row.get("title") or ""),
cate_name,
str(row.get("author") or ""),
str(row.get("category") or ""),
])))
return {
"title": str(row.get("title") or ""),
"summary": summary,
"canonical_text": "\n\n".join(filter(None, [str(row.get("title") or ""), summary, content])).strip(),
"keywords": ",".join(keywords),
"metadata_json": {
"cid": int(row.get("cid") or 0),
"cate_name": cate_name,
"author": str(row.get("author") or ""),
"category": str(row.get("category") or ""),
"article_id": int(row.get("article_id") or 0),
"published_at": str(row.get("published_at") or ""),
"is_video": int(row.get("is_video") or 0),
},
"source_created_at": int(row.get("create_time") or 0),
"source_updated_at": int(row.get("update_time") or row.get("create_time") or 0),
}
async def build_post_payload(self, source_id: int) -> Optional[Dict[str, Any]]:
row = await self.fetchone(
f"SELECT * FROM {self.table('community_post')} WHERE id=%s AND status=1 AND delete_time IS NULL LIMIT 1",
(source_id,),
)
if not row:
return None
tags = await self.fetchall(
f"""
SELECT t.name
FROM {self.table('community_post_tag')} pt
JOIN {self.table('community_tag')} t ON t.id = pt.tag_id
WHERE pt.post_id=%s
""",
(source_id,),
)
tag_names = [str(item.get("name") or "") for item in tags if item.get("name")]
ext = json_loads_maybe(row.get("ext"), {})
segments = [
str(row.get("content") or ""),
str(ext.get("translated_content") or "") if isinstance(ext, dict) else "",
str(ext.get("lottery_analysis_content") or "") if isinstance(ext, dict) else "",
" ".join(tag_names),
]
canonical = normalize_plain_text("\n\n".join(filter(None, segments)))
return {
"title": truncate_text(canonical, 40) or f"社区帖子#{source_id}",
"summary": truncate_text(canonical, 200),
"canonical_text": canonical,
"keywords": ",".join(dict.fromkeys(tag_names)),
"metadata_json": {
"post_type": int(row.get("post_type") or 0),
"match_id": int(row.get("match_id") or 0),
"is_paid": int(row.get("is_paid") or 0),
"user_id": int(row.get("user_id") or 0),
"tags": tag_names,
},
"source_created_at": int(row.get("create_time") or 0),
"source_updated_at": int(row.get("update_time") or row.get("create_time") or 0),
}
async def build_lottery_draw_result_payload(self, source_id: int) -> Optional[Dict[str, Any]]:
row = await self.fetchone(f"SELECT * FROM {self.table('lottery_draw_result')} WHERE id=%s LIMIT 1", (source_id,))
if not row:
return None
game = await self.fetchone(
f"SELECT name, template FROM {self.table('lottery_game')} WHERE code=%s LIMIT 1",
(row.get("code") or "",),
)
game = game or {}
numbers = [item for item in str(row.get("draw_code") or "").split(",") if item]
title = f"{game.get('name') or row.get('code')} 第{row.get('issue')}期开奖"
summary = f"{title} 于 {row.get('draw_time') or ''} 开奖，开奖号码为 {'、'.join(numbers)}。"
trend = json_loads_maybe(row.get("trend"), {})
canonical = "\n".join(filter(None, [
summary,
f"下一期期号：{row.get('next_issue')}" if row.get("next_issue") else "",
f"下期开奖时间：{row.get('next_time')}" if row.get("next_time") else "",
"走势数据：" + normalize_plain_text(json_text(trend)) if trend not in ({}, None) else "",
])).strip()
return {
"title": title,
"summary": summary,
"canonical_text": canonical,
"keywords": ",".join(filter(None, [
str(row.get("code") or ""),
str(game.get("name") or ""),
str(game.get("template") or ""),
str(row.get("issue") or ""),
])),
"metadata_json": {
"code": str(row.get("code") or ""),
"issue": str(row.get("issue") or ""),
"template": str(game.get("template") or ""),
"game_name": str(game.get("name") or ""),
"draw_time": str(row.get("draw_time") or ""),
},
"source_created_at": int(row.get("create_time") or 0),
"source_updated_at": int(row.get("update_time") or row.get("create_time") or 0),
}
async def build_lottery_ai_history_payload(self, source_id: int) -> Optional[Dict[str, Any]]:
row = await self.fetchone(f"SELECT * FROM {self.table('lottery_ai_analysis')} WHERE id=%s LIMIT 1", (source_id,))
if not row:
return None
result = row.get("result") if row.get("result") is not None else ""
result_text = result if isinstance(result, str) else json_text(result)
canonical = normalize_plain_text(result_text)
title = f"{row.get('code') or '彩票'} 第{row.get('issue') or '-'}期 {row.get('module') or 'module'} 历史AI分析"
return {
"title": title,
"summary": truncate_text(canonical, 220),
"canonical_text": canonical,
"keywords": ",".join(filter(None, [str(row.get("code") or ""), str(row.get("issue") or ""), str(row.get("module") or "")])),
"metadata_json": {
"code": str(row.get("code") or ""),
"issue": str(row.get("issue") or ""),
"module": str(row.get("module") or ""),
},
"source_created_at": int(row.get("create_time") or 0),
"source_updated_at": int(row.get("update_time") or row.get("create_time") or 0),
}
async def build_match_event_payload(self, source_id: int) -> Optional[Dict[str, Any]]:
row = await self.fetchone(f"SELECT * FROM {self.table('match')} WHERE id=%s AND is_show=1 LIMIT 1", (source_id,))
if not row:
return None
home = str(row.get("home_team") or "")
away = str(row.get("away_team") or "")
league = str(row.get("league_name") or "")
match_time_value = int(row.get("match_time") or 0)
match_time_text = time.strftime("%Y-%m-%d %H:%M", time.localtime(match_time_value)) if match_time_value > 0 else ""
history = await self.fetchall(
f"""
SELECT * FROM {self.table('match_history')}
WHERE (home_team=%s AND away_team=%s) OR (home_team=%s AND away_team=%s)
ORDER BY match_time DESC LIMIT 5
""",
(home, away, away, home),
)
home_recent = await self.fetchall(
f"SELECT * FROM {self.table('match_recent')} WHERE team_name=%s ORDER BY match_time DESC LIMIT 5",
(home,),
)
away_recent = await self.fetchall(
f"SELECT * FROM {self.table('match_recent')} WHERE team_name=%s ORDER BY match_time DESC LIMIT 5",
(away,),
)
title = f"{home} vs {away}".strip()
summary = (
f"{league} {title}，{home} 对阵 {away}，比赛时间 {match_time_text}，"
f"当前比分 {row.get('home_score') or 0}-{row.get('away_score') or 0}。"
)
canonical = "\n".join([
summary,
f"赔率：主胜{row.get('home_odds') or '-'}，平局{row.get('draw_odds') or '-'}，客胜{row.get('away_odds') or '-'}。",
"历史交锋：" + normalize_plain_text(json_text(history)),
f"{home}近期战绩：" + normalize_plain_text(json_text(home_recent)),
f"{away}近期战绩：" + normalize_plain_text(json_text(away_recent)),
])
return {
"title": title or f"赛事#{source_id}",
"summary": normalize_plain_text(summary),
"canonical_text": canonical.strip(),
"keywords": ",".join(filter(None, [league, home, away])),
"metadata_json": {
"league": league,
"teams": [team for team in [home, away] if team],
"match_time": match_time_text,
"status": int(row.get("status") or 0),
},
"source_created_at": int(row.get("create_time") or row.get("match_time") or 0),
"source_updated_at": int(row.get("update_time") or row.get("match_time") or row.get("create_time") or 0),
}
async def upsert_document(self, domain: str, subtype: str, source_id: int, embedder: "EmbeddingClient") -> Dict[str, Any]:
payload = await self.build_payload(domain, subtype, source_id)
if not payload:
await self.mark_document_inactive(domain, subtype, source_id)
return {"success": False, "error": "源内容不存在或不可用"}
content_hash = hashlib.md5(json_text([
payload["title"],
payload["summary"],
payload["canonical_text"],
payload["keywords"],
payload["metadata_json"],
]).encode("utf-8")).hexdigest()
existing = await self.fetchone(
f"""
SELECT * FROM {self.table('ai_kb_document')}
WHERE domain=%s AND subtype=%s AND source_id=%s LIMIT 1
""",
(domain, subtype, source_id),
)
if existing and str(existing.get("content_hash") or "") == content_hash and int(existing.get("is_active") or 0) == 1:
await self.update_document_metadata(int(existing["id"]), payload)
chunk_count = await self.fetchone(
f"SELECT COUNT(*) AS total FROM {self.table('ai_kb_chunk')} WHERE document_id=%s",
(int(existing["id"]),),
)
return {"success": True, "document_id": int(existing["id"]), "chunk_count": int((chunk_count or {}).get("total") or 0), "skipped": True}
chunks = split_text(str(payload["canonical_text"] or ""))
if not chunks:
chunks = [payload["summary"] or payload["title"] or "empty"]
embedding_result = await embedder.embed(chunks)
embeddings = embedding_result.get("embeddings") or []
if not embedding_result.get("success") or not embeddings:
return {"success": False, "error": embedding_result.get("error") or "embedding失败"}
await self.connect()
assert self.pool is not None
async with self.pool.acquire() as conn:
async with conn.cursor(aiomysql.DictCursor) as cur:
await conn.begin()
try:
metadata_json = json_text(payload["metadata_json"])
if existing:
document_id = int(existing["id"])
await cur.execute(
f"""
UPDATE {self.table('ai_kb_document')}
SET title=%s, summary=%s, canonical_text=%s, keywords=%s, metadata_json=%s,
source_created_at=%s, source_updated_at=%s, content_hash=%s, is_active=1, update_time=%s
WHERE id=%s
""",
(
payload["title"],
payload["summary"],
payload["canonical_text"],
payload["keywords"],
metadata_json,
int(payload["source_created_at"]),
int(payload["source_updated_at"]),
content_hash,
now_ts(),
document_id,
),
)
else:
await cur.execute(
f"""
INSERT INTO {self.table('ai_kb_document')}
(domain, subtype, source_id, title, summary, canonical_text, keywords, metadata_json,
source_created_at, source_updated_at, content_hash, is_active, create_time, update_time)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,1,%s,%s)
""",
(
domain,
subtype,
source_id,
payload["title"],
payload["summary"],
payload["canonical_text"],
payload["keywords"],
metadata_json,
int(payload["source_created_at"]),
int(payload["source_updated_at"]),
content_hash,
now_ts(),
now_ts(),
),
)
document_id = int(cur.lastrowid)
await cur.execute(f"DELETE FROM {self.table('ai_kb_chunk')} WHERE document_id=%s", (document_id,))
dimension = int(embedding_result.get("dimension") or len(embeddings[0] or []))
keyword_text = " ".join(filter(None, [
str(payload.get("title") or ""),
str(payload.get("summary") or ""),
str(payload.get("keywords") or ""),
json_text(payload.get("metadata_json") or {}),
]))
for index, chunk_text in enumerate(chunks):
embedding = [float(value) for value in embeddings[index]]
await cur.execute(
f"""
INSERT INTO {self.table('ai_kb_chunk')}
(document_id, collection_name, chunk_index, chunk_text, embedding_json,
embedding_model, embedding_dim, vector_norm, keyword_text, create_time, update_time)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
""",
(
document_id,
COLLECTION_NAME,
index,
chunk_text,
json_text(embedding),
embedding_result.get("model") or embedder.model,
dimension,
vector_norm(embedding),
keyword_text,
now_ts(),
now_ts(),
),
)
await conn.commit()
except Exception:
await conn.rollback()
raise
return {"success": True, "document_id": document_id, "chunk_count": len(chunks)}
async def update_document_metadata(self, document_id: int, payload: Dict[str, Any]) -> None:
await self.connect()
assert self.pool is not None
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(
f"""
UPDATE {self.table('ai_kb_document')}
SET title=%s, summary=%s, keywords=%s, metadata_json=%s,
source_created_at=%s, source_updated_at=%s, update_time=%s
WHERE id=%s
""",
(
payload["title"],
payload["summary"],
payload["keywords"],
json_text(payload["metadata_json"]),
int(payload["source_created_at"]),
int(payload["source_updated_at"]),
now_ts(),
document_id,
),
)
await conn.commit()
async def mark_document_inactive(self, domain: str, subtype: str, source_id: int) -> None:
await self.connect()
assert self.pool is not None
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(
f"""
UPDATE {self.table('ai_kb_document')}
SET is_active=0, update_time=%s
WHERE domain=%s AND subtype=%s AND source_id=%s
""",
(now_ts(), domain, subtype, source_id),
)
await conn.commit()
class EmbeddingClient:
def __init__(self):
self.api_key = os.environ.get("EMBEDDING_API_KEY") or os.environ.get("DASHSCOPE_API_KEY") or ""
self.base_url = (os.environ.get("EMBEDDING_BASE_URL") or DEFAULT_EMBEDDING_BASE_URL).rstrip("/")
self.model = os.environ.get("EMBEDDING_MODEL") or DEFAULT_EMBEDDING_MODEL
self.dimension = int(os.environ.get("EMBEDDING_DIM") or DEFAULT_EMBEDDING_DIM)
self.batch_size = max(1, int(os.environ.get("EMBEDDING_BATCH_SIZE") or 10))
async def embed(self, chunks: List[str]) -> Dict[str, Any]:
if not self.api_key or not self.base_url or not self.model:
return {"success": False, "error": "Embedding API配置未完成", "embeddings": []}
start = time.monotonic()
embeddings: List[List[float]] = []
tokens = 0
try:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=60)) as session:
for offset in range(0, len(chunks), self.batch_size):
batch = chunks[offset:offset + self.batch_size]
payload: Dict[str, Any] = {"model": self.model, "input": batch}
if self.dimension > 0:
payload["dimensions"] = self.dimension
async with session.post(
self.base_url + "/v1/embeddings",
json=payload,
headers={"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"},
ssl=False,
) as resp:
body = await resp.text()
try:
data = json.loads(body)
except Exception:
data = {}
if resp.status != 200 or not isinstance(data.get("data"), list):
message = ((data.get("error") or {}).get("message") if isinstance(data, dict) else "") or f"HTTP {resp.status}"
return {"success": False, "error": message, "embeddings": [], "cost_ms": int((time.monotonic() - start) * 1000)}
for row in data.get("data") or []:
embedding = row.get("embedding") if isinstance(row, dict) else None
if isinstance(embedding, list) and embedding:
embeddings.append([float(value) for value in embedding])
tokens += int(((data.get("usage") or {}).get("total_tokens") or 0) if isinstance(data, dict) else 0)
return {
"success": len(embeddings) == len(chunks),
"error": "" if len(embeddings) == len(chunks) else "Embedding结果数量不匹配",
"embeddings": embeddings,
"model": self.model,
"dimension": len(embeddings[0]) if embeddings else 0,
"tokens": tokens,
"cost_ms": int((time.monotonic() - start) * 1000),
}
except Exception as exc:
return {"success": False, "error": f"Embedding请求失败: {exc}", "embeddings": []}
class KbWorker:
def __init__(self, batch: int, block_ms: int, max_retries: int, worker_id: str):
self.batch = max(1, batch)
self.block_ms = max(100, block_ms)
self.max_retries = max(0, max_retries)
self.worker_id = worker_id
redis_cfg = get_config().redis
self.redis = aioredis.Redis(
host=redis_cfg.host,
port=redis_cfg.port,
password=redis_cfg.password or None,
db=redis_cfg.db,
decode_responses=True,
socket_connect_timeout=5,
socket_timeout=max(10, int(self.block_ms / 1000) + 5),
health_check_interval=30,
retry_on_timeout=True,
)
self.db = KbDatabase()
self.embedder = EmbeddingClient()
async def close(self) -> None:
await self.redis.close()
await self.db.close()
async def ensure_group(self) -> None:
try:
await self.redis.xgroup_create(STREAM_KEY, GROUP_NAME, id="0", mkstream=True)
logger.info("created redis stream group {} on {}", GROUP_NAME, STREAM_KEY)
except Exception as exc:
if "BUSYGROUP" not in str(exc):
raise
async def run_forever(self) -> None:
await self.ensure_group()
logger.info("KB worker started id={} batch={} block_ms={}", self.worker_id, self.batch, self.block_ms)
while True:
try:
await self.consume_once()
except (redis_exceptions.TimeoutError, redis_exceptions.ConnectionError, asyncio.TimeoutError) as exc:
logger.warning("redis read interrupted, continue polling: {}", exc)
await asyncio.sleep(1)
async def consume_once(self) -> int:
rows = await self.redis.xreadgroup(
GROUP_NAME,
self.worker_id,
streams={STREAM_KEY: ">"},
count=self.batch,
block=self.block_ms,
)
processed = 0
for _, messages in rows or []:
for message_id, fields in messages:
await self.handle_message(message_id, fields)
processed += 1
return processed
async def handle_message(self, message_id: str, fields: Dict[str, Any]) -> None:
domain = str(fields.get("domain") or "")
subtype = str(fields.get("subtype") or "")
source_id = int(fields.get("source_id") or 0)
attempts = int(fields.get("attempts") or 0)
started = time.monotonic()
try:
if not domain or not subtype or source_id <= 0:
raise ValueError("消息缺少 domain/subtype/source_id")
result = await self.db.upsert_document(domain, subtype, source_id, self.embedder)
if not result.get("success"):
raise RuntimeError(str(result.get("error") or "upsert failed"))
await self.redis.xack(STREAM_KEY, GROUP_NAME, message_id)
logger.info(
"kb upsert ok message={} source={}/{}/{} result={} elapsed={:.2f}s",
message_id,
domain,
subtype,
source_id,
result,
time.monotonic() - started,
)
except Exception as exc:
await self.handle_failure(message_id, fields, attempts, str(exc))
async def handle_failure(self, message_id: str, fields: Dict[str, Any], attempts: int, error: str) -> None:
next_attempt = attempts + 1
payload = dict(fields)
payload["attempts"] = str(next_attempt)
payload["last_error"] = error[:500]
payload["failed_at"] = str(now_ts())
if next_attempt > self.max_retries:
await self.redis.xadd(DEAD_STREAM_KEY, payload)
await self.redis.xack(STREAM_KEY, GROUP_NAME, message_id)
logger.error("kb message={} moved to dead stream after {} attempts: {}", message_id, next_attempt, error)
return
await self.redis.xadd(STREAM_KEY, payload)
await self.redis.xack(STREAM_KEY, GROUP_NAME, message_id)
logger.warning("kb message={} retry {}/{}: {}", message_id, next_attempt, self.max_retries, error)
def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Sport Era AI KB Redis Stream worker")
parser.add_argument("--batch", type=int, default=50)
parser.add_argument("--block-ms", type=int, default=5000)
parser.add_argument("--max-retries", type=int, default=3)
parser.add_argument("--worker-id", default="")
return parser.parse_args(argv)
async def async_main(argv: Optional[List[str]] = None) -> int:
args = parse_args(argv)
load_config()
worker_id = args.worker_id or f"{socket.gethostname()}-{os.getpid()}"
worker = KbWorker(args.batch, args.block_ms, args.max_retries, worker_id)
try:
await worker.run_forever()
finally:
await worker.close()
return 0
def main(argv: Optional[List[str]] = None) -> int:
return asyncio.run(async_main(argv))
if __name__ == "__main__":
raise SystemExit(main())