493 lines
20 KiB
Python
493 lines
20 KiB
Python
"""
|
|
懂球帝世界杯专属采集
|
|
|
|
- 固定采集 2026 世界杯 season_id=26123 / competition_id=61
|
|
- 赛程写入 la_match
|
|
- 分组积分榜写入 la_worldcup_standing
|
|
- 射手榜/助攻榜写入 la_worldcup_person_ranking
|
|
"""
|
|
import json
|
|
import re
|
|
import time
|
|
from datetime import datetime
|
|
from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse
|
|
|
|
import pymysql
|
|
import requests
|
|
|
|
SEASON_ID = 26123
|
|
COMPETITION_ID = 61
|
|
LEAGUE_NAME = "世界杯"
|
|
SPORT_TYPE = 1
|
|
LEAGUE_TAB_ID = 26123
|
|
LEAGUE_SORT = 285
|
|
CRAWLER_USER_AGENT = "Mozilla/5.0 (compatible; SportEra/1.0)"
|
|
BASE_PARAMS = {
|
|
"app": "dqd",
|
|
"version": "830",
|
|
"platform": "miniprogram",
|
|
"language": "zh-cn",
|
|
"app_type": "",
|
|
"from": "msite_com",
|
|
}
|
|
STANDING_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/standing"
|
|
SCHEDULE_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/schedule"
|
|
PERSON_RANKING_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/person_ranking"
|
|
ROUND_ORDER = [
|
|
"小组赛 第1轮",
|
|
"小组赛 第2轮",
|
|
"小组赛 第3轮",
|
|
"1/16决赛",
|
|
"1/8决赛",
|
|
"1/4决赛",
|
|
"半决赛",
|
|
"三四名决赛",
|
|
"决赛",
|
|
]
|
|
BRACKET_ROUNDS = ROUND_ORDER[3:]
|
|
|
|
|
|
def normalize_prefix(prefix: str) -> str:
|
|
prefix = prefix or "la_"
|
|
return prefix if re.match(r"^[A-Za-z0-9_]+$", prefix) else "la_"
|
|
|
|
|
|
def console_print(message: str) -> None:
|
|
try:
|
|
print(message)
|
|
except UnicodeEncodeError:
|
|
print(message.encode("utf-8", "backslashreplace").decode("utf-8"))
|
|
|
|
|
|
def get_headers() -> dict:
|
|
return {
|
|
"User-Agent": CRAWLER_USER_AGENT,
|
|
"Accept": "application/json, text/plain, */*",
|
|
"Referer": "https://www.dongqiudi.com/",
|
|
}
|
|
|
|
|
|
def infer_stage(round_name: str) -> str:
|
|
if round_name.startswith("小组赛"):
|
|
return "小组赛"
|
|
return "淘汰赛"
|
|
|
|
|
|
def match_status_to_int(status: str) -> int:
|
|
status = (status or "").lower()
|
|
status_map = {
|
|
"fixture": 0,
|
|
"notstarted": 0,
|
|
"played": 2,
|
|
"finished": 2,
|
|
"live": 1,
|
|
"playing": 1,
|
|
"postponed": 3,
|
|
"cancelled": 3,
|
|
}
|
|
return status_map.get(status, 0)
|
|
|
|
|
|
def round_weight(round_name: str) -> int:
|
|
if round_name in ROUND_ORDER:
|
|
return ROUND_ORDER.index(round_name)
|
|
match = re.search(r"小组赛\s*第(\d+)轮", round_name)
|
|
if match:
|
|
return int(match.group(1)) - 1
|
|
return 999
|
|
|
|
|
|
def build_url(base_url: str, params: dict) -> str:
|
|
parsed = urlparse(base_url)
|
|
query = dict(parse_qsl(parsed.query, keep_blank_values=True))
|
|
query.update({k: str(v) for k, v in params.items() if v is not None})
|
|
return urlunparse(parsed._replace(query=urlencode(query)))
|
|
|
|
|
|
def fetch_json(session: requests.Session, url: str, params: dict | None = None) -> dict:
|
|
resp = session.get(url, params=params, timeout=30)
|
|
resp.raise_for_status()
|
|
return resp.json()
|
|
|
|
|
|
def fetch_standings(session: requests.Session) -> dict:
|
|
return fetch_json(session, STANDING_URL, {"season_id": SEASON_ID, **BASE_PARAMS})
|
|
|
|
|
|
def fetch_schedule_index(session: requests.Session) -> dict:
|
|
return fetch_json(session, SCHEDULE_URL, {"season_id": SEASON_ID, **BASE_PARAMS})
|
|
|
|
|
|
def fetch_round_schedule(session: requests.Session, round_url: str) -> dict:
|
|
return fetch_json(session, build_url(round_url, BASE_PARAMS))
|
|
|
|
|
|
def fetch_person_ranking(session: requests.Session, rank_type: str) -> dict:
|
|
params = {"season_id": SEASON_ID, "type": rank_type, "app_type": "", **BASE_PARAMS}
|
|
return fetch_json(session, PERSON_RANKING_URL, params)
|
|
|
|
|
|
def parse_standings(payload: dict) -> tuple[str, list[dict]]:
|
|
rounds = payload.get("content", {}).get("rounds", []) or []
|
|
if not rounds:
|
|
return "", []
|
|
rows = []
|
|
for round_item in rounds:
|
|
content = round_item.get("content", {}) or {}
|
|
stage_name = content.get("name", "") or "小组赛"
|
|
for group in content.get("data", []) or []:
|
|
group_name = group.get("name", "") or ""
|
|
for row_order, team in enumerate(group.get("data", []) or [], start=1):
|
|
rows.append({
|
|
"season_id": SEASON_ID,
|
|
"stage_name": stage_name,
|
|
"group_name": group_name,
|
|
"team_id": int(team.get("team_id") or 0),
|
|
"team_name": str(team.get("team_name") or ""),
|
|
"team_logo": str(team.get("team_logo") or ""),
|
|
"rank": int(team.get("rank") or 0),
|
|
"points": int(team.get("points") or 0),
|
|
"played": int(team.get("matches_total") or 0),
|
|
"won": int(team.get("matches_won") or 0),
|
|
"drawn": int(team.get("matches_draw") or 0),
|
|
"lost": int(team.get("matches_lost") or 0),
|
|
"goals_for": int(team.get("goals_pro") or 0),
|
|
"goals_against": int(team.get("goals_against") or 0),
|
|
"goal_diff": int(team.get("goals_pro") or 0) - int(team.get("goals_against") or 0),
|
|
"row_order": row_order,
|
|
})
|
|
return rows[0]["stage_name"] if rows else "", rows
|
|
|
|
|
|
def parse_schedule_matches(schedule_index: dict, session: requests.Session) -> list[dict]:
|
|
rounds = schedule_index.get("content", {}).get("rounds", []) or []
|
|
matches = []
|
|
seen_match_ids = set()
|
|
for round_item in rounds:
|
|
round_name = round_item.get("name", "") or ""
|
|
round_url = round_item.get("url", "")
|
|
if not round_name or not round_url:
|
|
continue
|
|
round_payload = fetch_round_schedule(session, round_url)
|
|
round_matches = round_payload.get("content", {}).get("matches", []) or []
|
|
for item in round_matches:
|
|
match_id = int(item.get("match_id") or 0)
|
|
if not match_id or match_id in seen_match_ids:
|
|
continue
|
|
seen_match_ids.add(match_id)
|
|
start_play = str(item.get("start_play") or "")
|
|
match_time = 0
|
|
if start_play:
|
|
try:
|
|
match_time = int(datetime.strptime(start_play, "%Y-%m-%d %H:%M:%S").timestamp())
|
|
except Exception:
|
|
match_time = 0
|
|
matches.append({
|
|
"match_id": match_id,
|
|
"competition_id": int(item.get("competition_id") or COMPETITION_ID),
|
|
"home_team_id": str(item.get("team_A_id") or ""),
|
|
"away_team_id": str(item.get("team_B_id") or ""),
|
|
"league_name": LEAGUE_NAME,
|
|
"round_name": round_name,
|
|
"stage": infer_stage(round_name),
|
|
"home_team": str(item.get("team_A_name") or ""),
|
|
"home_icon": str(item.get("team_A_logo") or ""),
|
|
"home_score": int(item.get("fs_A") or 0),
|
|
"away_team": str(item.get("team_B_name") or ""),
|
|
"away_icon": str(item.get("team_B_logo") or ""),
|
|
"away_score": int(item.get("fs_B") or 0),
|
|
"sport_type": SPORT_TYPE,
|
|
"status": match_status_to_int(str(item.get("status") or "")),
|
|
"match_time": match_time,
|
|
"current_minute": str(item.get("minute") or ""),
|
|
"half_score": "",
|
|
"sort": round_weight(round_name),
|
|
})
|
|
matches.sort(key=lambda item: (round_weight(item["round_name"]), item["match_time"], item["match_id"]))
|
|
return matches
|
|
|
|
|
|
def parse_person_rankings(payload: dict, rank_type: str) -> tuple[list[str], list[dict]]:
|
|
content = payload.get("content", {}) or {}
|
|
header = content.get("header", []) or []
|
|
rows = []
|
|
for item in content.get("data", []) or []:
|
|
rows.append({
|
|
"season_id": SEASON_ID,
|
|
"rank_type": rank_type,
|
|
"person_id": int(item.get("person_id") or 0),
|
|
"person_name": str(item.get("person_name") or ""),
|
|
"person_logo": str(item.get("person_logo") or ""),
|
|
"team_id": int(item.get("team_id") or 0),
|
|
"team_name": str(item.get("team_name") or ""),
|
|
"team_logo": str(item.get("team_logo") or ""),
|
|
"rank": int(item.get("rank") or 0),
|
|
"count": int(item.get("count") or 0),
|
|
"extra_json": json.dumps(item, ensure_ascii=False),
|
|
})
|
|
return header, rows
|
|
|
|
|
|
def ensure_tables(conn, prefix: str) -> None:
|
|
standing_table = f"{prefix}worldcup_standing"
|
|
ranking_table = f"{prefix}worldcup_person_ranking"
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
f"""
|
|
CREATE TABLE IF NOT EXISTS `{standing_table}` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`season_id` INT NOT NULL DEFAULT 0,
|
|
`stage_name` VARCHAR(50) NOT NULL DEFAULT '',
|
|
`group_name` VARCHAR(50) NOT NULL DEFAULT '',
|
|
`team_id` BIGINT NOT NULL DEFAULT 0,
|
|
`team_name` VARCHAR(100) NOT NULL DEFAULT '',
|
|
`team_logo` VARCHAR(500) NOT NULL DEFAULT '',
|
|
`rank` INT NOT NULL DEFAULT 0,
|
|
`points` INT NOT NULL DEFAULT 0,
|
|
`played` INT NOT NULL DEFAULT 0,
|
|
`won` INT NOT NULL DEFAULT 0,
|
|
`drawn` INT NOT NULL DEFAULT 0,
|
|
`lost` INT NOT NULL DEFAULT 0,
|
|
`goals_for` INT NOT NULL DEFAULT 0,
|
|
`goals_against` INT NOT NULL DEFAULT 0,
|
|
`goal_diff` INT NOT NULL DEFAULT 0,
|
|
`row_order` INT NOT NULL DEFAULT 0,
|
|
`create_time` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`update_time` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `uk_worldcup_group_team` (`season_id`, `stage_name`, `group_name`, `team_id`),
|
|
KEY `idx_worldcup_group` (`season_id`, `stage_name`, `group_name`)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
"""
|
|
)
|
|
cur.execute(
|
|
f"""
|
|
CREATE TABLE IF NOT EXISTS `{ranking_table}` (
|
|
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
`season_id` INT NOT NULL DEFAULT 0,
|
|
`rank_type` VARCHAR(20) NOT NULL DEFAULT '',
|
|
`person_id` BIGINT NOT NULL DEFAULT 0,
|
|
`person_name` VARCHAR(100) NOT NULL DEFAULT '',
|
|
`person_logo` VARCHAR(500) NOT NULL DEFAULT '',
|
|
`team_id` BIGINT NOT NULL DEFAULT 0,
|
|
`team_name` VARCHAR(100) NOT NULL DEFAULT '',
|
|
`team_logo` VARCHAR(500) NOT NULL DEFAULT '',
|
|
`rank` INT NOT NULL DEFAULT 0,
|
|
`count` INT NOT NULL DEFAULT 0,
|
|
`extra_json` JSON NULL,
|
|
`create_time` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
`update_time` INT UNSIGNED NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (`id`),
|
|
UNIQUE KEY `uk_worldcup_rank_person` (`season_id`, `rank_type`, `person_id`),
|
|
KEY `idx_worldcup_rank_type` (`season_id`, `rank_type`, `rank`)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
|
"""
|
|
)
|
|
|
|
|
|
def ensure_worldcup_league(conn, prefix: str) -> None:
|
|
league_table = f"{prefix}league"
|
|
now = int(time.time())
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
f"SELECT id FROM `{league_table}` WHERE label=%s OR league_id=%s OR tab_id=%s LIMIT 1",
|
|
(LEAGUE_NAME, COMPETITION_ID, LEAGUE_TAB_ID),
|
|
)
|
|
row = cur.fetchone()
|
|
if row:
|
|
cur.execute(
|
|
f"""
|
|
UPDATE `{league_table}`
|
|
SET label=%s, `type`='league', sport_type=%s, sort=%s, is_show=1, league_id=%s, update_time=%s
|
|
WHERE id=%s
|
|
""",
|
|
(LEAGUE_NAME, SPORT_TYPE, LEAGUE_SORT, COMPETITION_ID, now, row["id"]),
|
|
)
|
|
return
|
|
|
|
cur.execute(
|
|
f"""
|
|
INSERT INTO `{league_table}`
|
|
(tab_id, league_id, label, `type`, sport_type, api, icon, sort, is_show, extra_data, sessionid, create_time, update_time)
|
|
VALUES (%s, %s, %s, 'league', %s, '', '', %s, 1, NULL, NULL, %s, %s)
|
|
""",
|
|
(LEAGUE_TAB_ID, COMPETITION_ID, LEAGUE_NAME, SPORT_TYPE, LEAGUE_SORT, now, now),
|
|
)
|
|
|
|
|
|
def sync_matches(conn, prefix: str, matches: list[dict]) -> tuple[int, int]:
|
|
table = f"{prefix}match"
|
|
now = int(time.time())
|
|
inserted = 0
|
|
updated = 0
|
|
with conn.cursor() as cur:
|
|
for item in matches:
|
|
cur.execute(
|
|
f"""
|
|
INSERT INTO `{table}`
|
|
(match_id, competition_id, home_team_id, away_team_id,
|
|
league_name, round_name, stage, league_icon,
|
|
home_team, home_icon, home_score,
|
|
away_team, away_icon, away_score,
|
|
sport_type, status, match_time,
|
|
current_minute, half_score,
|
|
is_hot, is_show, sort, create_time, update_time)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,'',%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,0,1,%s,%s,%s)
|
|
ON DUPLICATE KEY UPDATE
|
|
competition_id=VALUES(competition_id),
|
|
league_name=VALUES(league_name),
|
|
round_name=VALUES(round_name),
|
|
stage=VALUES(stage),
|
|
home_team=VALUES(home_team),
|
|
home_icon=VALUES(home_icon),
|
|
home_score=VALUES(home_score),
|
|
away_team=VALUES(away_team),
|
|
away_icon=VALUES(away_icon),
|
|
away_score=VALUES(away_score),
|
|
sport_type=VALUES(sport_type),
|
|
status=VALUES(status),
|
|
match_time=VALUES(match_time),
|
|
current_minute=VALUES(current_minute),
|
|
half_score=VALUES(half_score),
|
|
sort=VALUES(sort),
|
|
update_time=VALUES(update_time)
|
|
""",
|
|
(
|
|
item["match_id"],
|
|
item["competition_id"],
|
|
item["home_team_id"],
|
|
item["away_team_id"],
|
|
item["league_name"],
|
|
item["round_name"][:50],
|
|
item["stage"][:50],
|
|
item["home_team"][:100],
|
|
item["home_icon"][:255],
|
|
item["home_score"],
|
|
item["away_team"][:100],
|
|
item["away_icon"][:255],
|
|
item["away_score"],
|
|
item["sport_type"],
|
|
item["status"],
|
|
item["match_time"],
|
|
item["current_minute"][:20],
|
|
item["half_score"][:50],
|
|
item["sort"],
|
|
now,
|
|
now,
|
|
),
|
|
)
|
|
if cur.rowcount == 1:
|
|
inserted += 1
|
|
elif cur.rowcount == 2:
|
|
updated += 1
|
|
return inserted, updated
|
|
|
|
|
|
def sync_standings(conn, prefix: str, rows: list[dict]) -> int:
|
|
table = f"{prefix}worldcup_standing"
|
|
now = int(time.time())
|
|
count = 0
|
|
with conn.cursor() as cur:
|
|
for item in rows:
|
|
cur.execute(
|
|
f"""
|
|
INSERT INTO `{table}`
|
|
(season_id, stage_name, group_name, team_id, team_name, team_logo, `rank`,
|
|
points, played, won, drawn, lost, goals_for, goals_against, goal_diff,
|
|
row_order, create_time, update_time)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
|
ON DUPLICATE KEY UPDATE
|
|
team_name=VALUES(team_name),
|
|
team_logo=VALUES(team_logo),
|
|
`rank`=VALUES(`rank`),
|
|
points=VALUES(points),
|
|
played=VALUES(played),
|
|
won=VALUES(won),
|
|
drawn=VALUES(drawn),
|
|
lost=VALUES(lost),
|
|
goals_for=VALUES(goals_for),
|
|
goals_against=VALUES(goals_against),
|
|
goal_diff=VALUES(goal_diff),
|
|
row_order=VALUES(row_order),
|
|
update_time=VALUES(update_time)
|
|
""",
|
|
(
|
|
item["season_id"], item["stage_name"][:50], item["group_name"][:50], item["team_id"],
|
|
item["team_name"][:100], item["team_logo"][:500], item["rank"], item["points"],
|
|
item["played"], item["won"], item["drawn"], item["lost"], item["goals_for"],
|
|
item["goals_against"], item["goal_diff"], item["row_order"], now, now,
|
|
),
|
|
)
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def sync_rankings(conn, prefix: str, rows: list[dict], rank_type: str) -> int:
|
|
table = f"{prefix}worldcup_person_ranking"
|
|
now = int(time.time())
|
|
count = 0
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
f"DELETE FROM `{table}` WHERE season_id=%s AND rank_type=%s",
|
|
(SEASON_ID, rank_type),
|
|
)
|
|
for item in rows:
|
|
cur.execute(
|
|
f"""
|
|
INSERT INTO `{table}`
|
|
(season_id, rank_type, person_id, person_name, person_logo, team_id, team_name, team_logo,
|
|
`rank`, `count`, extra_json, create_time, update_time)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
|
""",
|
|
(
|
|
item["season_id"], item["rank_type"], item["person_id"], item["person_name"][:100],
|
|
item["person_logo"][:500], item["team_id"], item["team_name"][:100], item["team_logo"][:500],
|
|
item["rank"], item["count"], item["extra_json"], now, now,
|
|
),
|
|
)
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def run(db_config: dict):
|
|
db_config = dict(db_config)
|
|
prefix = normalize_prefix(db_config.pop("prefix", "la_"))
|
|
|
|
session = requests.Session()
|
|
session.headers.update(get_headers())
|
|
|
|
standings_payload = fetch_standings(session)
|
|
schedule_index = fetch_schedule_index(session)
|
|
goals_payload = fetch_person_ranking(session, "goals")
|
|
assists_payload = fetch_person_ranking(session, "assists")
|
|
|
|
stage_name, standings = parse_standings(standings_payload)
|
|
matches = parse_schedule_matches(schedule_index, session)
|
|
goal_header, goal_rows = parse_person_rankings(goals_payload, "goals")
|
|
assist_header, assist_rows = parse_person_rankings(assists_payload, "assists")
|
|
|
|
conn = pymysql.connect(**db_config, cursorclass=pymysql.cursors.DictCursor)
|
|
try:
|
|
ensure_tables(conn, prefix)
|
|
ensure_worldcup_league(conn, prefix)
|
|
inserted, updated = sync_matches(conn, prefix, matches)
|
|
standings_count = sync_standings(conn, prefix, standings)
|
|
goals_count = sync_rankings(conn, prefix, goal_rows, "goals")
|
|
assists_count = sync_rankings(conn, prefix, assist_rows, "assists")
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
console_print(f"世界杯阶段: {stage_name or '小组赛'}")
|
|
console_print(f"赛程轮次: {len(schedule_index.get('content', {}).get('rounds', []) or [])}, 比赛 {len(matches)} 场")
|
|
console_print(f"la_match: 新增 {inserted}, 更新 {updated}")
|
|
console_print(f"积分榜: {standings_count} 条")
|
|
console_print(f"射手榜: {goals_count} 条, 表头: {' / '.join(goal_header or [])}")
|
|
console_print(f"助攻榜: {assists_count} 条, 表头: {' / '.join(assist_header or [])}")
|
|
return {
|
|
"success": True,
|
|
"candidate_count": len(matches) + len(standings) + len(goal_rows) + len(assist_rows),
|
|
"saved_count": inserted + updated + standings_count + goals_count + assists_count,
|
|
"summary_text": f"世界杯赛程 {len(matches)} 场，la_match 新增 {inserted} 更新 {updated}，积分榜 {standings_count}，射手榜 {goals_count}，助攻榜 {assists_count}",
|
|
}
|