""" 懂球帝世界杯专属采集 - 固定采集 2026 世界杯 season_id=26123 / competition_id=61 - 赛程写入 la_match - 分组积分榜写入 la_worldcup_standing - 射手榜/助攻榜写入 la_worldcup_person_ranking """ import json import re import time from datetime import datetime from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse import pymysql import requests SEASON_ID = 26123 COMPETITION_ID = 61 LEAGUE_NAME = "世界杯" SPORT_TYPE = 1 LEAGUE_TAB_ID = 26123 LEAGUE_SORT = 285 CRAWLER_USER_AGENT = "Mozilla/5.0 (compatible; SportEra/1.0)" BASE_PARAMS = { "app": "dqd", "version": "830", "platform": "miniprogram", "language": "zh-cn", "app_type": "", "from": "msite_com", } STANDING_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/standing" SCHEDULE_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/schedule" PERSON_RANKING_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/person_ranking" ROUND_ORDER = [ "小组赛 第1轮", "小组赛 第2轮", "小组赛 第3轮", "1/16决赛", "1/8决赛", "1/4决赛", "半决赛", "三四名决赛", "决赛", ] BRACKET_ROUNDS = ROUND_ORDER[3:] def normalize_prefix(prefix: str) -> str: prefix = prefix or "la_" return prefix if re.match(r"^[A-Za-z0-9_]+$", prefix) else "la_" def console_print(message: str) -> None: try: print(message) except UnicodeEncodeError: print(message.encode("utf-8", "backslashreplace").decode("utf-8")) def get_headers() -> dict: return { "User-Agent": CRAWLER_USER_AGENT, "Accept": "application/json, text/plain, */*", "Referer": "https://www.dongqiudi.com/", } def infer_stage(round_name: str) -> str: if round_name.startswith("小组赛"): return "小组赛" return "淘汰赛" def match_status_to_int(status: str) -> int: status = (status or "").lower() status_map = { "fixture": 0, "notstarted": 0, "played": 2, "finished": 2, "live": 1, "playing": 1, "postponed": 3, "cancelled": 3, } return status_map.get(status, 0) def round_weight(round_name: str) -> int: if round_name in ROUND_ORDER: return ROUND_ORDER.index(round_name) match = re.search(r"小组赛\s*第(\d+)轮", round_name) if match: return int(match.group(1)) - 1 return 999 def build_url(base_url: str, params: dict) -> str: parsed = urlparse(base_url) query = dict(parse_qsl(parsed.query, keep_blank_values=True)) query.update({k: str(v) for k, v in params.items() if v is not None}) return urlunparse(parsed._replace(query=urlencode(query))) def fetch_json(session: requests.Session, url: str, params: dict | None = None) -> dict: resp = session.get(url, params=params, timeout=30) resp.raise_for_status() return resp.json() def fetch_standings(session: requests.Session) -> dict: return fetch_json(session, STANDING_URL, {"season_id": SEASON_ID, **BASE_PARAMS}) def fetch_schedule_index(session: requests.Session) -> dict: return fetch_json(session, SCHEDULE_URL, {"season_id": SEASON_ID, **BASE_PARAMS}) def fetch_round_schedule(session: requests.Session, round_url: str) -> dict: return fetch_json(session, build_url(round_url, BASE_PARAMS)) def fetch_person_ranking(session: requests.Session, rank_type: str) -> dict: params = {"season_id": SEASON_ID, "type": rank_type, "app_type": "", **BASE_PARAMS} return fetch_json(session, PERSON_RANKING_URL, params) def parse_standings(payload: dict) -> tuple[str, list[dict]]: rounds = payload.get("content", {}).get("rounds", []) or [] if not rounds: return "", [] rows = [] for round_item in rounds: content = round_item.get("content", {}) or {} stage_name = content.get("name", "") or "小组赛" for group in content.get("data", []) or []: group_name = group.get("name", "") or "" for row_order, team in enumerate(group.get("data", []) or [], start=1): rows.append({ "season_id": SEASON_ID, "stage_name": stage_name, "group_name": group_name, "team_id": int(team.get("team_id") or 0), "team_name": str(team.get("team_name") or ""), "team_logo": str(team.get("team_logo") or ""), "rank": int(team.get("rank") or 0), "points": int(team.get("points") or 0), "played": int(team.get("matches_total") or 0), "won": int(team.get("matches_won") or 0), "drawn": int(team.get("matches_draw") or 0), "lost": int(team.get("matches_lost") or 0), "goals_for": int(team.get("goals_pro") or 0), "goals_against": int(team.get("goals_against") or 0), "goal_diff": int(team.get("goals_pro") or 0) - int(team.get("goals_against") or 0), "row_order": row_order, }) return rows[0]["stage_name"] if rows else "", rows def parse_schedule_matches(schedule_index: dict, session: requests.Session) -> list[dict]: rounds = schedule_index.get("content", {}).get("rounds", []) or [] matches = [] seen_match_ids = set() for round_item in rounds: round_name = round_item.get("name", "") or "" round_url = round_item.get("url", "") if not round_name or not round_url: continue round_payload = fetch_round_schedule(session, round_url) round_matches = round_payload.get("content", {}).get("matches", []) or [] for item in round_matches: match_id = int(item.get("match_id") or 0) if not match_id or match_id in seen_match_ids: continue seen_match_ids.add(match_id) start_play = str(item.get("start_play") or "") match_time = 0 if start_play: try: match_time = int(datetime.strptime(start_play, "%Y-%m-%d %H:%M:%S").timestamp()) except Exception: match_time = 0 matches.append({ "match_id": match_id, "competition_id": int(item.get("competition_id") or COMPETITION_ID), "home_team_id": str(item.get("team_A_id") or ""), "away_team_id": str(item.get("team_B_id") or ""), "league_name": LEAGUE_NAME, "round_name": round_name, "stage": infer_stage(round_name), "home_team": str(item.get("team_A_name") or ""), "home_icon": str(item.get("team_A_logo") or ""), "home_score": int(item.get("fs_A") or 0), "away_team": str(item.get("team_B_name") or ""), "away_icon": str(item.get("team_B_logo") or ""), "away_score": int(item.get("fs_B") or 0), "sport_type": SPORT_TYPE, "status": match_status_to_int(str(item.get("status") or "")), "match_time": match_time, "current_minute": str(item.get("minute") or ""), "half_score": "", "sort": round_weight(round_name), }) matches.sort(key=lambda item: (round_weight(item["round_name"]), item["match_time"], item["match_id"])) return matches def parse_person_rankings(payload: dict, rank_type: str) -> tuple[list[str], list[dict]]: content = payload.get("content", {}) or {} header = content.get("header", []) or [] rows = [] for item in content.get("data", []) or []: rows.append({ "season_id": SEASON_ID, "rank_type": rank_type, "person_id": int(item.get("person_id") or 0), "person_name": str(item.get("person_name") or ""), "person_logo": str(item.get("person_logo") or ""), "team_id": int(item.get("team_id") or 0), "team_name": str(item.get("team_name") or ""), "team_logo": str(item.get("team_logo") or ""), "rank": int(item.get("rank") or 0), "count": int(item.get("count") or 0), "extra_json": json.dumps(item, ensure_ascii=False), }) return header, rows def ensure_tables(conn, prefix: str) -> None: standing_table = f"{prefix}worldcup_standing" ranking_table = f"{prefix}worldcup_person_ranking" with conn.cursor() as cur: cur.execute( f""" CREATE TABLE IF NOT EXISTS `{standing_table}` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `season_id` INT NOT NULL DEFAULT 0, `stage_name` VARCHAR(50) NOT NULL DEFAULT '', `group_name` VARCHAR(50) NOT NULL DEFAULT '', `team_id` BIGINT NOT NULL DEFAULT 0, `team_name` VARCHAR(100) NOT NULL DEFAULT '', `team_logo` VARCHAR(500) NOT NULL DEFAULT '', `rank` INT NOT NULL DEFAULT 0, `points` INT NOT NULL DEFAULT 0, `played` INT NOT NULL DEFAULT 0, `won` INT NOT NULL DEFAULT 0, `drawn` INT NOT NULL DEFAULT 0, `lost` INT NOT NULL DEFAULT 0, `goals_for` INT NOT NULL DEFAULT 0, `goals_against` INT NOT NULL DEFAULT 0, `goal_diff` INT NOT NULL DEFAULT 0, `row_order` INT NOT NULL DEFAULT 0, `create_time` INT UNSIGNED NOT NULL DEFAULT 0, `update_time` INT UNSIGNED NOT NULL DEFAULT 0, PRIMARY KEY (`id`), UNIQUE KEY `uk_worldcup_group_team` (`season_id`, `stage_name`, `group_name`, `team_id`), KEY `idx_worldcup_group` (`season_id`, `stage_name`, `group_name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 """ ) cur.execute( f""" CREATE TABLE IF NOT EXISTS `{ranking_table}` ( `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, `season_id` INT NOT NULL DEFAULT 0, `rank_type` VARCHAR(20) NOT NULL DEFAULT '', `person_id` BIGINT NOT NULL DEFAULT 0, `person_name` VARCHAR(100) NOT NULL DEFAULT '', `person_logo` VARCHAR(500) NOT NULL DEFAULT '', `team_id` BIGINT NOT NULL DEFAULT 0, `team_name` VARCHAR(100) NOT NULL DEFAULT '', `team_logo` VARCHAR(500) NOT NULL DEFAULT '', `rank` INT NOT NULL DEFAULT 0, `count` INT NOT NULL DEFAULT 0, `extra_json` JSON NULL, `create_time` INT UNSIGNED NOT NULL DEFAULT 0, `update_time` INT UNSIGNED NOT NULL DEFAULT 0, PRIMARY KEY (`id`), UNIQUE KEY `uk_worldcup_rank_person` (`season_id`, `rank_type`, `person_id`), KEY `idx_worldcup_rank_type` (`season_id`, `rank_type`, `rank`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 """ ) def ensure_worldcup_league(conn, prefix: str) -> None: league_table = f"{prefix}league" now = int(time.time()) with conn.cursor() as cur: cur.execute( f"SELECT id FROM `{league_table}` WHERE label=%s OR league_id=%s OR tab_id=%s LIMIT 1", (LEAGUE_NAME, COMPETITION_ID, LEAGUE_TAB_ID), ) row = cur.fetchone() if row: cur.execute( f""" UPDATE `{league_table}` SET label=%s, `type`='league', sport_type=%s, sort=%s, is_show=1, league_id=%s, update_time=%s WHERE id=%s """, (LEAGUE_NAME, SPORT_TYPE, LEAGUE_SORT, COMPETITION_ID, now, row["id"]), ) return cur.execute( f""" INSERT INTO `{league_table}` (tab_id, league_id, label, `type`, sport_type, api, icon, sort, is_show, extra_data, sessionid, create_time, update_time) VALUES (%s, %s, %s, 'league', %s, '', '', %s, 1, NULL, NULL, %s, %s) """, (LEAGUE_TAB_ID, COMPETITION_ID, LEAGUE_NAME, SPORT_TYPE, LEAGUE_SORT, now, now), ) def sync_matches(conn, prefix: str, matches: list[dict]) -> tuple[int, int]: table = f"{prefix}match" now = int(time.time()) inserted = 0 updated = 0 with conn.cursor() as cur: for item in matches: cur.execute( f""" INSERT INTO `{table}` (match_id, competition_id, home_team_id, away_team_id, league_name, round_name, stage, league_icon, home_team, home_icon, home_score, away_team, away_icon, away_score, sport_type, status, match_time, current_minute, half_score, is_hot, is_show, sort, create_time, update_time) VALUES (%s,%s,%s,%s,%s,%s,%s,'',%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,0,1,%s,%s,%s) ON DUPLICATE KEY UPDATE competition_id=VALUES(competition_id), league_name=VALUES(league_name), round_name=VALUES(round_name), stage=VALUES(stage), home_team=VALUES(home_team), home_icon=VALUES(home_icon), home_score=VALUES(home_score), away_team=VALUES(away_team), away_icon=VALUES(away_icon), away_score=VALUES(away_score), sport_type=VALUES(sport_type), status=VALUES(status), match_time=VALUES(match_time), current_minute=VALUES(current_minute), half_score=VALUES(half_score), sort=VALUES(sort), update_time=VALUES(update_time) """, ( item["match_id"], item["competition_id"], item["home_team_id"], item["away_team_id"], item["league_name"], item["round_name"][:50], item["stage"][:50], item["home_team"][:100], item["home_icon"][:255], item["home_score"], item["away_team"][:100], item["away_icon"][:255], item["away_score"], item["sport_type"], item["status"], item["match_time"], item["current_minute"][:20], item["half_score"][:50], item["sort"], now, now, ), ) if cur.rowcount == 1: inserted += 1 elif cur.rowcount == 2: updated += 1 return inserted, updated def sync_standings(conn, prefix: str, rows: list[dict]) -> int: table = f"{prefix}worldcup_standing" now = int(time.time()) count = 0 with conn.cursor() as cur: for item in rows: cur.execute( f""" INSERT INTO `{table}` (season_id, stage_name, group_name, team_id, team_name, team_logo, `rank`, points, played, won, drawn, lost, goals_for, goals_against, goal_diff, row_order, create_time, update_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE team_name=VALUES(team_name), team_logo=VALUES(team_logo), `rank`=VALUES(`rank`), points=VALUES(points), played=VALUES(played), won=VALUES(won), drawn=VALUES(drawn), lost=VALUES(lost), goals_for=VALUES(goals_for), goals_against=VALUES(goals_against), goal_diff=VALUES(goal_diff), row_order=VALUES(row_order), update_time=VALUES(update_time) """, ( item["season_id"], item["stage_name"][:50], item["group_name"][:50], item["team_id"], item["team_name"][:100], item["team_logo"][:500], item["rank"], item["points"], item["played"], item["won"], item["drawn"], item["lost"], item["goals_for"], item["goals_against"], item["goal_diff"], item["row_order"], now, now, ), ) count += 1 return count def sync_rankings(conn, prefix: str, rows: list[dict], rank_type: str) -> int: table = f"{prefix}worldcup_person_ranking" now = int(time.time()) count = 0 with conn.cursor() as cur: cur.execute( f"DELETE FROM `{table}` WHERE season_id=%s AND rank_type=%s", (SEASON_ID, rank_type), ) for item in rows: cur.execute( f""" INSERT INTO `{table}` (season_id, rank_type, person_id, person_name, person_logo, team_id, team_name, team_logo, `rank`, `count`, extra_json, create_time, update_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) """, ( item["season_id"], item["rank_type"], item["person_id"], item["person_name"][:100], item["person_logo"][:500], item["team_id"], item["team_name"][:100], item["team_logo"][:500], item["rank"], item["count"], item["extra_json"], now, now, ), ) count += 1 return count def run(db_config: dict): db_config = dict(db_config) prefix = normalize_prefix(db_config.pop("prefix", "la_")) session = requests.Session() session.headers.update(get_headers()) standings_payload = fetch_standings(session) schedule_index = fetch_schedule_index(session) goals_payload = fetch_person_ranking(session, "goals") assists_payload = fetch_person_ranking(session, "assists") stage_name, standings = parse_standings(standings_payload) matches = parse_schedule_matches(schedule_index, session) goal_header, goal_rows = parse_person_rankings(goals_payload, "goals") assist_header, assist_rows = parse_person_rankings(assists_payload, "assists") conn = pymysql.connect(**db_config, cursorclass=pymysql.cursors.DictCursor) try: ensure_tables(conn, prefix) ensure_worldcup_league(conn, prefix) inserted, updated = sync_matches(conn, prefix, matches) standings_count = sync_standings(conn, prefix, standings) goals_count = sync_rankings(conn, prefix, goal_rows, "goals") assists_count = sync_rankings(conn, prefix, assist_rows, "assists") conn.commit() finally: conn.close() console_print(f"世界杯阶段: {stage_name or '小组赛'}") console_print(f"赛程轮次: {len(schedule_index.get('content', {}).get('rounds', []) or [])}, 比赛 {len(matches)} 场") console_print(f"la_match: 新增 {inserted}, 更新 {updated}") console_print(f"积分榜: {standings_count} 条") console_print(f"射手榜: {goals_count} 条, 表头: {' / '.join(goal_header or [])}") console_print(f"助攻榜: {assists_count} 条, 表头: {' / '.join(assist_header or [])}") return { "success": True, "candidate_count": len(matches) + len(standings) + len(goal_rows) + len(assist_rows), "saved_count": inserted + updated + standings_count + goals_count + assists_count, "summary_text": f"世界杯赛程 {len(matches)} 场，la_match 新增 {inserted} 更新 {updated}，积分榜 {standings_count}，射手榜 {goals_count}，助攻榜 {assists_count}", }