迁移目录

2026-06-12 22:27:18 +08:00
parent 7381097582
commit cd44cd6e47
92 changed files with 1839 additions and 0 deletions
@@ -0,0 +1,492 @@
+"""
+懂球帝世界杯专属采集
+
+- 固定采集 2026 世界杯 season_id=26123 / competition_id=61
+- 赛程写入 la_match
+- 分组积分榜写入 la_worldcup_standing
+- 射手榜/助攻榜写入 la_worldcup_person_ranking
+"""
+import json
+import re
+import time
+from datetime import datetime
+from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse
+
+import pymysql
+import requests
+
+SEASON_ID = 26123
+COMPETITION_ID = 61
+LEAGUE_NAME = "世界杯"
+SPORT_TYPE = 1
+LEAGUE_TAB_ID = 26123
+LEAGUE_SORT = 285
+CRAWLER_USER_AGENT = "Mozilla/5.0 (compatible; SportEra/1.0)"
+BASE_PARAMS = {
+    "app": "dqd",
+    "version": "830",
+    "platform": "miniprogram",
+    "language": "zh-cn",
+    "app_type": "",
+    "from": "msite_com",
+}
+STANDING_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/standing"
+SCHEDULE_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/schedule"
+PERSON_RANKING_URL = "https://sport-data.dongqiudi.com/soccer/biz/data/person_ranking"
+ROUND_ORDER = [
+    "小组赛 第1轮",
+    "小组赛 第2轮",
+    "小组赛 第3轮",
+    "1/16决赛",
+    "1/8决赛",
+    "1/4决赛",
+    "半决赛",
+    "三四名决赛",
+    "决赛",
+]
+BRACKET_ROUNDS = ROUND_ORDER[3:]
+
+
+def normalize_prefix(prefix: str) -> str:
+    prefix = prefix or "la_"
+    return prefix if re.match(r"^[A-Za-z0-9_]+$", prefix) else "la_"
+
+
+def console_print(message: str) -> None:
+    try:
+        print(message)
+    except UnicodeEncodeError:
+        print(message.encode("utf-8", "backslashreplace").decode("utf-8"))
+
+
+def get_headers() -> dict:
+    return {
+        "User-Agent": CRAWLER_USER_AGENT,
+        "Accept": "application/json, text/plain, */*",
+        "Referer": "https://www.dongqiudi.com/",
+    }
+
+
+def infer_stage(round_name: str) -> str:
+    if round_name.startswith("小组赛"):
+        return "小组赛"
+    return "淘汰赛"
+
+
+def match_status_to_int(status: str) -> int:
+    status = (status or "").lower()
+    status_map = {
+        "fixture": 0,
+        "notstarted": 0,
+        "played": 2,
+        "finished": 2,
+        "live": 1,
+        "playing": 1,
+        "postponed": 3,
+        "cancelled": 3,
+    }
+    return status_map.get(status, 0)
+
+
+def round_weight(round_name: str) -> int:
+    if round_name in ROUND_ORDER:
+        return ROUND_ORDER.index(round_name)
+    match = re.search(r"小组赛\s*第(\d+)轮", round_name)
+    if match:
+        return int(match.group(1)) - 1
+    return 999
+
+
+def build_url(base_url: str, params: dict) -> str:
+    parsed = urlparse(base_url)
+    query = dict(parse_qsl(parsed.query, keep_blank_values=True))
+    query.update({k: str(v) for k, v in params.items() if v is not None})
+    return urlunparse(parsed._replace(query=urlencode(query)))
+
+
+def fetch_json(session: requests.Session, url: str, params: dict | None = None) -> dict:
+    resp = session.get(url, params=params, timeout=30)
+    resp.raise_for_status()
+    return resp.json()
+
+
+def fetch_standings(session: requests.Session) -> dict:
+    return fetch_json(session, STANDING_URL, {"season_id": SEASON_ID, **BASE_PARAMS})
+
+
+def fetch_schedule_index(session: requests.Session) -> dict:
+    return fetch_json(session, SCHEDULE_URL, {"season_id": SEASON_ID, **BASE_PARAMS})
+
+
+def fetch_round_schedule(session: requests.Session, round_url: str) -> dict:
+    return fetch_json(session, build_url(round_url, BASE_PARAMS))
+
+
+def fetch_person_ranking(session: requests.Session, rank_type: str) -> dict:
+    params = {"season_id": SEASON_ID, "type": rank_type, "app_type": "", **BASE_PARAMS}
+    return fetch_json(session, PERSON_RANKING_URL, params)
+
+
+def parse_standings(payload: dict) -> tuple[str, list[dict]]:
+    rounds = payload.get("content", {}).get("rounds", []) or []
+    if not rounds:
+        return "", []
+    rows = []
+    for round_item in rounds:
+        content = round_item.get("content", {}) or {}
+        stage_name = content.get("name", "") or "小组赛"
+        for group in content.get("data", []) or []:
+            group_name = group.get("name", "") or ""
+            for row_order, team in enumerate(group.get("data", []) or [], start=1):
+                rows.append({
+                    "season_id": SEASON_ID,
+                    "stage_name": stage_name,
+                    "group_name": group_name,
+                    "team_id": int(team.get("team_id") or 0),
+                    "team_name": str(team.get("team_name") or ""),
+                    "team_logo": str(team.get("team_logo") or ""),
+                    "rank": int(team.get("rank") or 0),
+                    "points": int(team.get("points") or 0),
+                    "played": int(team.get("matches_total") or 0),
+                    "won": int(team.get("matches_won") or 0),
+                    "drawn": int(team.get("matches_draw") or 0),
+                    "lost": int(team.get("matches_lost") or 0),
+                    "goals_for": int(team.get("goals_pro") or 0),
+                    "goals_against": int(team.get("goals_against") or 0),
+                    "goal_diff": int(team.get("goals_pro") or 0) - int(team.get("goals_against") or 0),
+                    "row_order": row_order,
+                })
+    return rows[0]["stage_name"] if rows else "", rows
+
+
+def parse_schedule_matches(schedule_index: dict, session: requests.Session) -> list[dict]:
+    rounds = schedule_index.get("content", {}).get("rounds", []) or []
+    matches = []
+    seen_match_ids = set()
+    for round_item in rounds:
+        round_name = round_item.get("name", "") or ""
+        round_url = round_item.get("url", "")
+        if not round_name or not round_url:
+            continue
+        round_payload = fetch_round_schedule(session, round_url)
+        round_matches = round_payload.get("content", {}).get("matches", []) or []
+        for item in round_matches:
+            match_id = int(item.get("match_id") or 0)
+            if not match_id or match_id in seen_match_ids:
+                continue
+            seen_match_ids.add(match_id)
+            start_play = str(item.get("start_play") or "")
+            match_time = 0
+            if start_play:
+                try:
+                    match_time = int(datetime.strptime(start_play, "%Y-%m-%d %H:%M:%S").timestamp())
+                except Exception:
+                    match_time = 0
+            matches.append({
+                "match_id": match_id,
+                "competition_id": int(item.get("competition_id") or COMPETITION_ID),
+                "home_team_id": str(item.get("team_A_id") or ""),
+                "away_team_id": str(item.get("team_B_id") or ""),
+                "league_name": LEAGUE_NAME,
+                "round_name": round_name,
+                "stage": infer_stage(round_name),
+                "home_team": str(item.get("team_A_name") or ""),
+                "home_icon": str(item.get("team_A_logo") or ""),
+                "home_score": int(item.get("fs_A") or 0),
+                "away_team": str(item.get("team_B_name") or ""),
+                "away_icon": str(item.get("team_B_logo") or ""),
+                "away_score": int(item.get("fs_B") or 0),
+                "sport_type": SPORT_TYPE,
+                "status": match_status_to_int(str(item.get("status") or "")),
+                "match_time": match_time,
+                "current_minute": str(item.get("minute") or ""),
+                "half_score": "",
+                "sort": round_weight(round_name),
+            })
+    matches.sort(key=lambda item: (round_weight(item["round_name"]), item["match_time"], item["match_id"]))
+    return matches
+
+
+def parse_person_rankings(payload: dict, rank_type: str) -> tuple[list[str], list[dict]]:
+    content = payload.get("content", {}) or {}
+    header = content.get("header", []) or []
+    rows = []
+    for item in content.get("data", []) or []:
+        rows.append({
+            "season_id": SEASON_ID,
+            "rank_type": rank_type,
+            "person_id": int(item.get("person_id") or 0),
+            "person_name": str(item.get("person_name") or ""),
+            "person_logo": str(item.get("person_logo") or ""),
+            "team_id": int(item.get("team_id") or 0),
+            "team_name": str(item.get("team_name") or ""),
+            "team_logo": str(item.get("team_logo") or ""),
+            "rank": int(item.get("rank") or 0),
+            "count": int(item.get("count") or 0),
+            "extra_json": json.dumps(item, ensure_ascii=False),
+        })
+    return header, rows
+
+
+def ensure_tables(conn, prefix: str) -> None:
+    standing_table = f"{prefix}worldcup_standing"
+    ranking_table = f"{prefix}worldcup_person_ranking"
+    with conn.cursor() as cur:
+        cur.execute(
+            f"""
+            CREATE TABLE IF NOT EXISTS `{standing_table}` (
+              `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+              `season_id` INT NOT NULL DEFAULT 0,
+              `stage_name` VARCHAR(50) NOT NULL DEFAULT '',
+              `group_name` VARCHAR(50) NOT NULL DEFAULT '',
+              `team_id` BIGINT NOT NULL DEFAULT 0,
+              `team_name` VARCHAR(100) NOT NULL DEFAULT '',
+              `team_logo` VARCHAR(500) NOT NULL DEFAULT '',
+              `rank` INT NOT NULL DEFAULT 0,
+              `points` INT NOT NULL DEFAULT 0,
+              `played` INT NOT NULL DEFAULT 0,
+              `won` INT NOT NULL DEFAULT 0,
+              `drawn` INT NOT NULL DEFAULT 0,
+              `lost` INT NOT NULL DEFAULT 0,
+              `goals_for` INT NOT NULL DEFAULT 0,
+              `goals_against` INT NOT NULL DEFAULT 0,
+              `goal_diff` INT NOT NULL DEFAULT 0,
+              `row_order` INT NOT NULL DEFAULT 0,
+              `create_time` INT UNSIGNED NOT NULL DEFAULT 0,
+              `update_time` INT UNSIGNED NOT NULL DEFAULT 0,
+              PRIMARY KEY (`id`),
+              UNIQUE KEY `uk_worldcup_group_team` (`season_id`, `stage_name`, `group_name`, `team_id`),
+              KEY `idx_worldcup_group` (`season_id`, `stage_name`, `group_name`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+            """
+        )
+        cur.execute(
+            f"""
+            CREATE TABLE IF NOT EXISTS `{ranking_table}` (
+              `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+              `season_id` INT NOT NULL DEFAULT 0,
+              `rank_type` VARCHAR(20) NOT NULL DEFAULT '',
+              `person_id` BIGINT NOT NULL DEFAULT 0,
+              `person_name` VARCHAR(100) NOT NULL DEFAULT '',
+              `person_logo` VARCHAR(500) NOT NULL DEFAULT '',
+              `team_id` BIGINT NOT NULL DEFAULT 0,
+              `team_name` VARCHAR(100) NOT NULL DEFAULT '',
+              `team_logo` VARCHAR(500) NOT NULL DEFAULT '',
+              `rank` INT NOT NULL DEFAULT 0,
+              `count` INT NOT NULL DEFAULT 0,
+              `extra_json` JSON NULL,
+              `create_time` INT UNSIGNED NOT NULL DEFAULT 0,
+              `update_time` INT UNSIGNED NOT NULL DEFAULT 0,
+              PRIMARY KEY (`id`),
+              UNIQUE KEY `uk_worldcup_rank_person` (`season_id`, `rank_type`, `person_id`),
+              KEY `idx_worldcup_rank_type` (`season_id`, `rank_type`, `rank`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+            """
+        )
+
+
+def ensure_worldcup_league(conn, prefix: str) -> None:
+    league_table = f"{prefix}league"
+    now = int(time.time())
+    with conn.cursor() as cur:
+        cur.execute(
+            f"SELECT id FROM `{league_table}` WHERE label=%s OR league_id=%s OR tab_id=%s LIMIT 1",
+            (LEAGUE_NAME, COMPETITION_ID, LEAGUE_TAB_ID),
+        )
+        row = cur.fetchone()
+        if row:
+            cur.execute(
+                f"""
+                UPDATE `{league_table}`
+                SET label=%s, `type`='league', sport_type=%s, sort=%s, is_show=1, league_id=%s, update_time=%s
+                WHERE id=%s
+                """,
+                (LEAGUE_NAME, SPORT_TYPE, LEAGUE_SORT, COMPETITION_ID, now, row["id"]),
+            )
+            return
+
+        cur.execute(
+            f"""
+            INSERT INTO `{league_table}`
+            (tab_id, league_id, label, `type`, sport_type, api, icon, sort, is_show, extra_data, sessionid, create_time, update_time)
+            VALUES (%s, %s, %s, 'league', %s, '', '', %s, 1, NULL, NULL, %s, %s)
+            """,
+            (LEAGUE_TAB_ID, COMPETITION_ID, LEAGUE_NAME, SPORT_TYPE, LEAGUE_SORT, now, now),
+        )
+
+
+def sync_matches(conn, prefix: str, matches: list[dict]) -> tuple[int, int]:
+    table = f"{prefix}match"
+    now = int(time.time())
+    inserted = 0
+    updated = 0
+    with conn.cursor() as cur:
+        for item in matches:
+            cur.execute(
+                f"""
+                INSERT INTO `{table}`
+                (match_id, competition_id, home_team_id, away_team_id,
+                 league_name, round_name, stage, league_icon,
+                 home_team, home_icon, home_score,
+                 away_team, away_icon, away_score,
+                 sport_type, status, match_time,
+                 current_minute, half_score,
+                 is_hot, is_show, sort, create_time, update_time)
+                VALUES (%s,%s,%s,%s,%s,%s,%s,'',%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,0,1,%s,%s,%s)
+                ON DUPLICATE KEY UPDATE
+                competition_id=VALUES(competition_id),
+                league_name=VALUES(league_name),
+                round_name=VALUES(round_name),
+                stage=VALUES(stage),
+                home_team=VALUES(home_team),
+                home_icon=VALUES(home_icon),
+                home_score=VALUES(home_score),
+                away_team=VALUES(away_team),
+                away_icon=VALUES(away_icon),
+                away_score=VALUES(away_score),
+                sport_type=VALUES(sport_type),
+                status=VALUES(status),
+                match_time=VALUES(match_time),
+                current_minute=VALUES(current_minute),
+                half_score=VALUES(half_score),
+                sort=VALUES(sort),
+                update_time=VALUES(update_time)
+                """,
+                (
+                    item["match_id"],
+                    item["competition_id"],
+                    item["home_team_id"],
+                    item["away_team_id"],
+                    item["league_name"],
+                    item["round_name"][:50],
+                    item["stage"][:50],
+                    item["home_team"][:100],
+                    item["home_icon"][:255],
+                    item["home_score"],
+                    item["away_team"][:100],
+                    item["away_icon"][:255],
+                    item["away_score"],
+                    item["sport_type"],
+                    item["status"],
+                    item["match_time"],
+                    item["current_minute"][:20],
+                    item["half_score"][:50],
+                    item["sort"],
+                    now,
+                    now,
+                ),
+            )
+            if cur.rowcount == 1:
+                inserted += 1
+            elif cur.rowcount == 2:
+                updated += 1
+    return inserted, updated
+
+
+def sync_standings(conn, prefix: str, rows: list[dict]) -> int:
+    table = f"{prefix}worldcup_standing"
+    now = int(time.time())
+    count = 0
+    with conn.cursor() as cur:
+        for item in rows:
+            cur.execute(
+                f"""
+                INSERT INTO `{table}`
+                (season_id, stage_name, group_name, team_id, team_name, team_logo, `rank`,
+                 points, played, won, drawn, lost, goals_for, goals_against, goal_diff,
+                 row_order, create_time, update_time)
+                VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
+                ON DUPLICATE KEY UPDATE
+                team_name=VALUES(team_name),
+                team_logo=VALUES(team_logo),
+                `rank`=VALUES(`rank`),
+                points=VALUES(points),
+                played=VALUES(played),
+                won=VALUES(won),
+                drawn=VALUES(drawn),
+                lost=VALUES(lost),
+                goals_for=VALUES(goals_for),
+                goals_against=VALUES(goals_against),
+                goal_diff=VALUES(goal_diff),
+                row_order=VALUES(row_order),
+                update_time=VALUES(update_time)
+                """,
+                (
+                    item["season_id"], item["stage_name"][:50], item["group_name"][:50], item["team_id"],
+                    item["team_name"][:100], item["team_logo"][:500], item["rank"], item["points"],
+                    item["played"], item["won"], item["drawn"], item["lost"], item["goals_for"],
+                    item["goals_against"], item["goal_diff"], item["row_order"], now, now,
+                ),
+            )
+            count += 1
+    return count
+
+
+def sync_rankings(conn, prefix: str, rows: list[dict], rank_type: str) -> int:
+    table = f"{prefix}worldcup_person_ranking"
+    now = int(time.time())
+    count = 0
+    with conn.cursor() as cur:
+        cur.execute(
+            f"DELETE FROM `{table}` WHERE season_id=%s AND rank_type=%s",
+            (SEASON_ID, rank_type),
+        )
+        for item in rows:
+            cur.execute(
+                f"""
+                INSERT INTO `{table}`
+                (season_id, rank_type, person_id, person_name, person_logo, team_id, team_name, team_logo,
+                 `rank`, `count`, extra_json, create_time, update_time)
+                VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
+                """,
+                (
+                    item["season_id"], item["rank_type"], item["person_id"], item["person_name"][:100],
+                    item["person_logo"][:500], item["team_id"], item["team_name"][:100], item["team_logo"][:500],
+                    item["rank"], item["count"], item["extra_json"], now, now,
+                ),
+            )
+            count += 1
+    return count
+
+
+def run(db_config: dict):
+    db_config = dict(db_config)
+    prefix = normalize_prefix(db_config.pop("prefix", "la_"))
+
+    session = requests.Session()
+    session.headers.update(get_headers())
+
+    standings_payload = fetch_standings(session)
+    schedule_index = fetch_schedule_index(session)
+    goals_payload = fetch_person_ranking(session, "goals")
+    assists_payload = fetch_person_ranking(session, "assists")
+
+    stage_name, standings = parse_standings(standings_payload)
+    matches = parse_schedule_matches(schedule_index, session)
+    goal_header, goal_rows = parse_person_rankings(goals_payload, "goals")
+    assist_header, assist_rows = parse_person_rankings(assists_payload, "assists")
+
+    conn = pymysql.connect(**db_config, cursorclass=pymysql.cursors.DictCursor)
+    try:
+        ensure_tables(conn, prefix)
+        ensure_worldcup_league(conn, prefix)
+        inserted, updated = sync_matches(conn, prefix, matches)
+        standings_count = sync_standings(conn, prefix, standings)
+        goals_count = sync_rankings(conn, prefix, goal_rows, "goals")
+        assists_count = sync_rankings(conn, prefix, assist_rows, "assists")
+        conn.commit()
+    finally:
+        conn.close()
+
+    console_print(f"世界杯阶段: {stage_name or '小组赛'}")
+    console_print(f"赛程轮次: {len(schedule_index.get('content', {}).get('rounds', []) or [])}, 比赛 {len(matches)} 场")
+    console_print(f"la_match: 新增 {inserted}, 更新 {updated}")
+    console_print(f"积分榜: {standings_count} 条")
+    console_print(f"射手榜: {goals_count} 条, 表头: {' / '.join(goal_header or [])}")
+    console_print(f"助攻榜: {assists_count} 条, 表头: {' / '.join(assist_header or [])}")
+    return {
+        "success": True,
+        "candidate_count": len(matches) + len(standings) + len(goal_rows) + len(assist_rows),
+        "saved_count": inserted + updated + standings_count + goals_count + assists_count,
+        "summary_text": f"世界杯赛程 {len(matches)} 场，la_match 新增 {inserted} 更新 {updated}，积分榜 {standings_count}，射手榜 {goals_count}，助攻榜 {assists_count}",
+    }