deploy: auto commit docker changes 2026-07-09 22:54:47

docker/.env.docker.example

@@ -21,6 +21,14 @@ AI_KB_REDIS_DB=4
 DQD_LOG_LEVEL=INFO
 DQD_LOG_FILE=/app/logs/crawler.log
 
+# Third-party match odds crawler. Keep the real account in docker/.env.docker only.
+ODDS_USERNAME=change-me
+ODDS_PASSWORD=change-me
+ODDS_BASE_URLS=https://205.201.0.120/,https://mos022.com/,https://hga039.com/,https://mos100.com/
+ODDS_SPORT_TYPES=FT,BK,ES,TN,VB,BM,TT,BS,SK,OP
+ODDS_SHOW_TYPES=live,today,early
+ODDS_VERIFY_TLS=0
+
 # AI KB worker embedding config. Keep real key in docker/.env.docker only.
 EMBEDDING_API_KEY=change-me
 EMBEDDING_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode

docker/crawler/config/crawler_tasks.yaml

@@ -159,3 +159,8 @@ tasks:
     action: match_live_stream
     cron: "*/1 * * * *"
     active: true
+  - key: match_odds
+    name: 第三方赛事赔率抓取
+    action: match_odds
+    cron: "*/5 * * * *"
+    active: true

docker/crawler/docker/entrypoint.sh

@@ -19,7 +19,7 @@ from pathlib import Path
 env_file = Path("/etc/sport-era-crawler/env.sh")
 with env_file.open("w", encoding="utf-8") as fp:
     for key, value in sorted(os.environ.items()):
-        if key.startswith("DQD_") or key in {"TZ", "PYTHONPATH", "PYTHONUNBUFFERED"}:
+        if key.startswith(("DQD_", "ODDS_", "HG_ODDS_")) or key in {"TZ", "PYTHONPATH", "PYTHONUNBUFFERED"}:
             fp.write(f"export {key}={shlex.quote(value)}\n")
 env_file.chmod(0o600)
 PY

docker/crawler/main.py

@@ -31,6 +31,7 @@
   python main.py cba_news           # 采集CBA新闻资讯
   python main.py ai_comment_dispatch # Docker AI 评论调度
   python main.py match_live_stream  # 抓取赛事直播线路播放流
+  python main.py match_odds        # 抓取第三方赛事赔率
   python main.py article_content_fetch # 抓取资讯文章正文内容
   python main.py all                # 采集积分榜 + 赛程
   python main.py single <season_id> # 采集单个联赛积分榜
@@ -61,6 +62,7 @@ TASK_ALERT_POLICY = {
     "lottery_draw_force": "saved_if_candidates",
     "ai_comment_dispatch": "saved_if_candidates",
     "match_live_stream": "saved_if_candidates",
+    "match_odds": "saved_if_candidates",
 }
 
 DEFAULT_ACTION_TIMEOUT = 600
@@ -89,6 +91,7 @@ ACTION_TIMEOUTS = {
     "dqd_worldcup": 600,
     "ai_comment_dispatch": 1800,
     "match_live_stream": 180,
+    "match_odds": 300,
 }
 
 
@@ -543,6 +546,29 @@ async def cmd_match_live_stream():
     return await asyncio.to_thread(run, db_config)
 
 
+async def cmd_match_odds():
+    import asyncio
+    from match_odds import run
+    from src.core.config import load_config
+
+    cfg = load_config().database
+    db_config = {
+        "host": cfg.host,
+        "port": cfg.port,
+        "user": cfg.username,
+        "password": cfg.password,
+        "database": cfg.database,
+        "charset": cfg.charset,
+        "prefix": cfg.prefix,
+    }
+    result = await asyncio.to_thread(run, db_config)
+    prefix = "✅" if result.get("success") else "❌"
+    print(f"{prefix} {result.get('summary_text', '第三方赛事赔率抓取完成')}")
+    if result.get("error"):
+        print(f"   {result['error']}")
+    return result
+
+
 async def cmd_match_data():
     from src.scheduler.task_runner import TaskRunner
     async with TaskRunner() as runner:
@@ -859,6 +885,7 @@ def main():
         "cba_news": cmd_cba_news,
         "ai_comment_dispatch": cmd_ai_comment_dispatch,
         "match_live_stream": cmd_match_live_stream,
+        "match_odds": cmd_match_odds,
         "article_content_fetch": cmd_article_content_fetch,
         "match_finish": cmd_match_finish,
         "live_detail": cmd_live_detail,

docker/crawler/match_odds.py

@@ -0,0 +1,643 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import base64
+import json
+import os
+import re
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Optional
+from urllib.parse import urljoin
+from xml.etree import ElementTree as ET
+
+import pymysql
+import requests
+import urllib3
+
+
+DEFAULT_BASE_URLS = [
+    "https://205.201.0.120/",
+    "https://mos022.com/",
+    "https://hga039.com/",
+    "https://mos100.com/",
+]
+DEFAULT_SPORT_TYPES = ["FT", "BK", "ES", "TN", "VB", "BM", "TT", "BS", "SK", "OP"]
+DEFAULT_SHOW_TYPES = ["live", "today", "early"]
+SHOW_TYPE_RTYPE = {
+    "live": "rb",
+    "today": "r",
+    "early": "r",
+}
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/137.0.0.0 Safari/537.36"
+)
+REQUEST_DELAY_SECONDS = 0.4
+
+
+class OddsGameListUnavailable(Exception):
+    """Raised when the account/menu combination has no accessible game list."""
+
+
+def _split_env_list(value: str, default: Iterable[str]) -> List[str]:
+    if not str(value or "").strip():
+        return list(default)
+    parts = re.split(r"[\s,]+", str(value).strip())
+    return [item.strip() for item in parts if item.strip()]
+
+
+def _normalize_base_url(url: str) -> str:
+    normalized = str(url or "").strip()
+    if not normalized:
+        raise ValueError("empty odds base url")
+    if not normalized.startswith(("http://", "https://")):
+        normalized = "https://" + normalized
+    return normalized.rstrip("/") + "/"
+
+
+def _bool_env(value: Any, default: bool = False) -> bool:
+    if value is None or value == "":
+        return default
+    return str(value).strip().lower() in {"1", "true", "yes", "on", "y"}
+
+
+def _safe_int(value: Any, default: int = 0) -> int:
+    try:
+        return int(str(value).strip())
+    except (TypeError, ValueError):
+        return default
+
+
+def _text(raw: Dict[str, str], key: str) -> str:
+    return str(raw.get(key) or "").strip()
+
+
+def _limit(value: Any, size: int) -> str:
+    return re.sub(r"\s+", " ", str(value or "")).strip()[:size]
+
+
+def _dump_json(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
+
+
+def _extract_version(html: str) -> str:
+    match = re.search(r"top\.ver\s*=\s*'([^']+)'", html or "")
+    if not match:
+        raise ValueError("odds site version not found")
+    return match.group(1)
+
+
+def _parse_login_xml(body: str) -> Dict[str, str]:
+    root = ET.fromstring((body or "").strip())
+    status = (root.findtext("status") or "").strip()
+    msg = (root.findtext("msg") or "").strip()
+    uid = (root.findtext("uid") or "").strip()
+    if status != "200" or not uid:
+        code_message = (root.findtext("code_message") or root.findtext("msg") or "login failed").strip()
+        raise ValueError(f"odds login failed: {code_message or msg or status}")
+    return {child.tag: (child.text or "").strip() for child in list(root)}
+
+
+def _skippable_game_list_reason(body: str) -> str:
+    prefix = re.sub(r"\s+", " ", str(body or "").strip())[:120]
+    if prefix.startswith("CheckEMNU"):
+        return "菜单不可用"
+    if prefix.startswith("coupon_date error"):
+        return "早盘日期不可用"
+    return ""
+
+
+def _build_markets(raw: Dict[str, str]) -> Dict[str, Any]:
+    return {
+        "handicap": {
+            "line": _text(raw, "RATIO_R"),
+            "home": _text(raw, "IOR_RH"),
+            "away": _text(raw, "IOR_RC"),
+            "strong": _text(raw, "STRONG"),
+        },
+        "total": {
+            "over_line": _text(raw, "RATIO_OUO"),
+            "under_line": _text(raw, "RATIO_OUU"),
+            "over": _text(raw, "IOR_OUC"),
+            "under": _text(raw, "IOR_OUH"),
+        },
+        "moneyline": {
+            "home": _text(raw, "IOR_MH"),
+            "away": _text(raw, "IOR_MC"),
+            "draw": _text(raw, "IOR_MN"),
+        },
+        "first_half": {
+            "handicap_line": _text(raw, "RATIO_HR"),
+            "handicap_home": _text(raw, "IOR_HRH"),
+            "handicap_away": _text(raw, "IOR_HRC"),
+            "total_over_line": _text(raw, "RATIO_HOUO"),
+            "total_under_line": _text(raw, "RATIO_HOUU"),
+            "total_over": _text(raw, "IOR_HOUC"),
+            "total_under": _text(raw, "IOR_HOUH"),
+            "home_win": _text(raw, "IOR_HMH"),
+            "away_win": _text(raw, "IOR_HMC"),
+            "draw": _text(raw, "IOR_HMN"),
+        },
+        "both_teams_to_score": {
+            "yes": _text(raw, "IOR_TSY"),
+            "no": _text(raw, "IOR_TSN"),
+        },
+        "kickoff": {
+            "home": _text(raw, "IOR_TKH"),
+            "away": _text(raw, "IOR_TKC"),
+        },
+    }
+
+
+def parse_game_list_xml(
+    body: str,
+    *,
+    source_site: str,
+    source_url: str,
+    sport_type: str,
+    show_type: str,
+    odds_type: str,
+) -> List[Dict[str, Any]]:
+    root = ET.fromstring((body or "").strip())
+    items: List[Dict[str, Any]] = []
+    ec_nodes = root.findall(".//ec")
+    if not ec_nodes:
+        ec_nodes = [root]
+
+    for ec_node in ec_nodes:
+        ec_id = str(ec_node.attrib.get("id") or "").replace("ec", "", 1)
+        for game in ec_node.findall(".//game"):
+            raw = {child.tag: (child.text or "").strip() for child in list(game)}
+            source_match_id = _text(raw, "GID") or str(game.attrib.get("id") or "").strip()
+            if not source_match_id:
+                continue
+            if not _text(raw, "ECID") and ec_id:
+                raw["ECID"] = ec_id
+            markets = _build_markets(raw)
+            item = {
+                "source_site": _limit(source_site, 50),
+                "source_url": _limit(source_url, 255),
+                "source_match_id": _limit(source_match_id, 64),
+                "source_parent_id": _limit(_text(raw, "GIDM"), 64),
+                "source_league_id": _limit(_text(raw, "LID"), 64),
+                "source_event_id": _limit(_text(raw, "ECID") or ec_id, 64),
+                "sport_type": _limit(str(sport_type).upper(), 20),
+                "show_type": _limit(str(show_type).lower(), 20),
+                "odds_type": _limit(str(odds_type).upper(), 20),
+                "league_name": _limit(_text(raw, "LEAGUE"), 150),
+                "match_time_text": _limit(_text(raw, "DATETIME"), 50),
+                "home_team_id": _limit(_text(raw, "TEAM_H_ID"), 64),
+                "away_team_id": _limit(_text(raw, "TEAM_C_ID"), 64),
+                "home_team": _limit(_text(raw, "TEAM_H"), 150),
+                "away_team": _limit(_text(raw, "TEAM_C"), 150),
+                "strong_side": _limit(_text(raw, "STRONG"), 10),
+                "more_count": _safe_int(_text(raw, "MORE")),
+                "is_running": 1 if _text(raw, "IS_RB").upper() == "Y" or str(show_type).lower() == "live" else 0,
+                "handicap_line": _limit(markets["handicap"]["line"], 32),
+                "handicap_home_odds": _limit(markets["handicap"]["home"], 32),
+                "handicap_away_odds": _limit(markets["handicap"]["away"], 32),
+                "total_line": _limit(markets["total"]["over_line"], 32),
+                "total_over_odds": _limit(markets["total"]["over"], 32),
+                "total_under_odds": _limit(markets["total"]["under"], 32),
+                "home_win_odds": _limit(markets["moneyline"]["home"], 32),
+                "draw_odds": _limit(markets["moneyline"]["draw"], 32),
+                "away_win_odds": _limit(markets["moneyline"]["away"], 32),
+                "half_handicap_line": _limit(markets["first_half"]["handicap_line"], 32),
+                "half_home_win_odds": _limit(markets["first_half"]["home_win"], 32),
+                "half_draw_odds": _limit(markets["first_half"]["draw"], 32),
+                "half_away_win_odds": _limit(markets["first_half"]["away_win"], 32),
+                "raw_data": raw,
+                "markets": markets,
+            }
+            items.append(item)
+    return items
+
+
+@dataclass
+class OddsLoginSession:
+    base_url: str
+    transform_url: str
+    version: str
+    uid: str
+    ltype: str
+    odds_type: str
+
+
+class HgOddsClient:
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        username: str,
+        password: str,
+        user_agent: str = DEFAULT_USER_AGENT,
+        timeout: int = 20,
+        verify_tls: bool = False,
+        langx: str = "zh-cn",
+        odds_type: str = "H",
+    ) -> None:
+        self.base_url = _normalize_base_url(base_url)
+        self.username = username
+        self.password = password
+        self.user_agent = user_agent
+        self.timeout = int(timeout)
+        self.verify_tls = bool(verify_tls)
+        self.langx = langx
+        self.odds_type = odds_type
+        self.session = requests.Session()
+        self.session.verify = self.verify_tls
+        self.session.headers.update(
+            {
+                "User-Agent": self.user_agent,
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Referer": self.base_url,
+            }
+        )
+        if not self.verify_tls:
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+        self.login_session: Optional[OddsLoginSession] = None
+
+    def login(self) -> OddsLoginSession:
+        bootstrap = self.session.post(
+            self.base_url,
+            data={"detection": "Y", "sub_doubleLogin": "", "isapp": "", "q": "", "appversion": ""},
+            timeout=self.timeout,
+        )
+        bootstrap.raise_for_status()
+        version = _extract_version(bootstrap.text)
+        transform_url = urljoin(self.base_url, "transform_nl.php") + f"?ver={version}"
+        user_agent_b64 = base64.b64encode(self.user_agent.encode("utf-8")).decode("ascii")
+        login_response = self.session.post(
+            transform_url,
+            data={
+                "p": "chk_login",
+                "langx": self.langx,
+                "ver": version,
+                "username": self.username,
+                "password": self.password,
+                "app": "N",
+                "auto": "EDACFD",
+                "userAgent": user_agent_b64,
+            },
+            timeout=self.timeout,
+        )
+        login_response.raise_for_status()
+        parsed = _parse_login_xml(login_response.text)
+        self.login_session = OddsLoginSession(
+            base_url=self.base_url,
+            transform_url=transform_url,
+            version=version,
+            uid=parsed["uid"],
+            ltype=parsed.get("ltype") or "3",
+            odds_type=parsed.get("odd_f_type") or self.odds_type,
+        )
+        return self.login_session
+
+    def fetch_game_list(self, sport_type: str, show_type: str) -> List[Dict[str, Any]]:
+        login_session = self.login_session or self.login()
+        normalized_show_type = str(show_type).lower()
+        normalized_sport_type = str(sport_type).upper()
+        now_ms = int(time.time() * 1000)
+        response = self.session.post(
+            login_session.transform_url,
+            data={
+                "uid": login_session.uid,
+                "ver": login_session.version,
+                "langx": self.langx,
+                "p": "get_game_list",
+                "p3type": "",
+                "date": "",
+                "gtype": normalized_sport_type.lower(),
+                "showtype": normalized_show_type,
+                "rtype": SHOW_TYPE_RTYPE.get(normalized_show_type, "r"),
+                "ltype": login_session.ltype,
+                "filter": normalized_sport_type,
+                "cupFantasy": "N",
+                "sorttype": "L",
+                "specialClick": "",
+                "isFantasy": "N",
+                "ts": str(now_ms),
+                "chgSortTS": str(now_ms),
+            },
+            timeout=self.timeout,
+        )
+        response.raise_for_status()
+        body = response.text or ""
+        skip_reason = _skippable_game_list_reason(body)
+        if skip_reason:
+            raise OddsGameListUnavailable(skip_reason)
+        if not body.lstrip().startswith("<"):
+            prefix = re.sub(r"\s+", " ", body.strip())[:120]
+            raise ValueError(f"unexpected odds response: {prefix}")
+        return parse_game_list_xml(
+            body,
+            source_site="hg",
+            source_url=login_session.base_url,
+            sport_type=normalized_sport_type,
+            show_type=normalized_show_type,
+            odds_type=login_session.odds_type,
+        )
+
+
+@dataclass
+class MatchOddsRepository:
+    db_config: Dict[str, Any]
+
+    def __post_init__(self) -> None:
+        self.prefix = str(self.db_config.get("prefix") or "la_")
+        self.conn = pymysql.connect(
+            host=self.db_config["host"],
+            port=int(self.db_config["port"]),
+            user=self.db_config["user"],
+            password=self.db_config["password"],
+            database=self.db_config["database"],
+            charset=self.db_config.get("charset") or "utf8mb4",
+            cursorclass=pymysql.cursors.DictCursor,
+            autocommit=False,
+        )
+
+    def close(self) -> None:
+        self.conn.close()
+
+    def ensure_table(self) -> None:
+        sql = f"""
+        CREATE TABLE IF NOT EXISTS `{self.prefix}match_odds` (
+          `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+          `source_site` VARCHAR(50) NOT NULL DEFAULT '' COMMENT '赔率来源',
+          `source_url` VARCHAR(255) NOT NULL DEFAULT '' COMMENT '来源线路',
+          `source_match_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '来源比赛ID',
+          `source_parent_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '来源主比赛ID',
+          `source_league_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '来源联赛ID',
+          `source_event_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '来源事件ID',
+          `sport_type` VARCHAR(20) NOT NULL DEFAULT '' COMMENT '运动类型',
+          `show_type` VARCHAR(20) NOT NULL DEFAULT '' COMMENT '盘口类型 live/today/early',
+          `odds_type` VARCHAR(20) NOT NULL DEFAULT '' COMMENT '赔率格式',
+          `league_name` VARCHAR(150) NOT NULL DEFAULT '' COMMENT '联赛名称',
+          `match_time_text` VARCHAR(50) NOT NULL DEFAULT '' COMMENT '来源比赛时间文本',
+          `home_team_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '主队来源ID',
+          `away_team_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '客队来源ID',
+          `home_team` VARCHAR(150) NOT NULL DEFAULT '' COMMENT '主队名称',
+          `away_team` VARCHAR(150) NOT NULL DEFAULT '' COMMENT '客队名称',
+          `strong_side` VARCHAR(10) NOT NULL DEFAULT '' COMMENT '让球强弱方',
+          `more_count` INT UNSIGNED NOT NULL DEFAULT 0 COMMENT '更多玩法数量',
+          `is_running` TINYINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '是否滚球',
+          `handicap_line` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '让球盘口',
+          `handicap_home_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '主队让球赔率',
+          `handicap_away_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '客队让球赔率',
+          `total_line` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '大小球盘口',
+          `total_over_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '大球赔率',
+          `total_under_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '小球赔率',
+          `home_win_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '主胜赔率',
+          `draw_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '平局赔率',
+          `away_win_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '客胜赔率',
+          `half_handicap_line` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '上半场让球盘口',
+          `half_home_win_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '上半场主胜赔率',
+          `half_draw_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '上半场平局赔率',
+          `half_away_win_odds` VARCHAR(32) NOT NULL DEFAULT '' COMMENT '上半场客胜赔率',
+          `create_time` INT UNSIGNED NOT NULL DEFAULT 0,
+          `raw_data` JSON NULL COMMENT '来源原始game节点',
+          `markets_json` JSON NULL COMMENT '规范化盘口JSON',
+          `update_time` INT UNSIGNED NOT NULL DEFAULT 0,
+          PRIMARY KEY (`id`),
+          UNIQUE KEY `uk_source_match` (`source_site`, `source_match_id`),
+          KEY `idx_sport_show` (`sport_type`, `show_type`),
+          KEY `idx_league` (`source_league_id`),
+          KEY `idx_update_time` (`update_time`)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='第三方赛事赔率表'
+        """
+        with self.conn.cursor() as cur:
+            cur.execute(sql)
+        self.conn.commit()
+
+    def upsert_latest(self, items: List[Dict[str, Any]], now_ts: Optional[int] = None) -> int:
+        if not items:
+            return 0
+        now_ts = int(now_ts or time.time())
+        sql = (
+            f"INSERT INTO `{self.prefix}match_odds` ("
+            "`source_site`, `source_url`, `source_match_id`, `source_parent_id`, `source_league_id`, `source_event_id`, "
+            "`sport_type`, `show_type`, `odds_type`, `league_name`, `match_time_text`, "
+            "`home_team_id`, `away_team_id`, `home_team`, `away_team`, `strong_side`, `more_count`, `is_running`, "
+            "`handicap_line`, `handicap_home_odds`, `handicap_away_odds`, "
+            "`total_line`, `total_over_odds`, `total_under_odds`, "
+            "`home_win_odds`, `draw_odds`, `away_win_odds`, "
+            "`half_handicap_line`, `half_home_win_odds`, `half_draw_odds`, `half_away_win_odds`, "
+            "`create_time`, `raw_data`, `markets_json`, `update_time`"
+            ") VALUES ("
+            + ", ".join(["%s"] * 35)
+            + ") ON DUPLICATE KEY UPDATE "
+            "`source_url`=VALUES(`source_url`), "
+            "`source_parent_id`=VALUES(`source_parent_id`), "
+            "`source_league_id`=VALUES(`source_league_id`), "
+            "`source_event_id`=VALUES(`source_event_id`), "
+            "`sport_type`=VALUES(`sport_type`), "
+            "`show_type`=VALUES(`show_type`), "
+            "`odds_type`=VALUES(`odds_type`), "
+            "`league_name`=VALUES(`league_name`), "
+            "`match_time_text`=VALUES(`match_time_text`), "
+            "`home_team_id`=VALUES(`home_team_id`), "
+            "`away_team_id`=VALUES(`away_team_id`), "
+            "`home_team`=VALUES(`home_team`), "
+            "`away_team`=VALUES(`away_team`), "
+            "`strong_side`=VALUES(`strong_side`), "
+            "`more_count`=VALUES(`more_count`), "
+            "`is_running`=VALUES(`is_running`), "
+            "`handicap_line`=VALUES(`handicap_line`), "
+            "`handicap_home_odds`=VALUES(`handicap_home_odds`), "
+            "`handicap_away_odds`=VALUES(`handicap_away_odds`), "
+            "`total_line`=VALUES(`total_line`), "
+            "`total_over_odds`=VALUES(`total_over_odds`), "
+            "`total_under_odds`=VALUES(`total_under_odds`), "
+            "`home_win_odds`=VALUES(`home_win_odds`), "
+            "`draw_odds`=VALUES(`draw_odds`), "
+            "`away_win_odds`=VALUES(`away_win_odds`), "
+            "`half_handicap_line`=VALUES(`half_handicap_line`), "
+            "`half_home_win_odds`=VALUES(`half_home_win_odds`), "
+            "`half_draw_odds`=VALUES(`half_draw_odds`), "
+            "`half_away_win_odds`=VALUES(`half_away_win_odds`), "
+            "`raw_data`=VALUES(`raw_data`), "
+            "`markets_json`=VALUES(`markets_json`), "
+            "`update_time`=VALUES(`update_time`)"
+        )
+        saved_count = 0
+        with self.conn.cursor() as cur:
+            for item in items:
+                params = (
+                    item["source_site"],
+                    item["source_url"],
+                    item["source_match_id"],
+                    item["source_parent_id"],
+                    item["source_league_id"],
+                    item["source_event_id"],
+                    item["sport_type"],
+                    item["show_type"],
+                    item["odds_type"],
+                    item["league_name"],
+                    item["match_time_text"],
+                    item["home_team_id"],
+                    item["away_team_id"],
+                    item["home_team"],
+                    item["away_team"],
+                    item["strong_side"],
+                    int(item["more_count"]),
+                    int(item["is_running"]),
+                    item["handicap_line"],
+                    item["handicap_home_odds"],
+                    item["handicap_away_odds"],
+                    item["total_line"],
+                    item["total_over_odds"],
+                    item["total_under_odds"],
+                    item["home_win_odds"],
+                    item["draw_odds"],
+                    item["away_win_odds"],
+                    item["half_handicap_line"],
+                    item["half_home_win_odds"],
+                    item["half_draw_odds"],
+                    item["half_away_win_odds"],
+                    now_ts,
+                    _dump_json(item["raw_data"]),
+                    _dump_json(item["markets"]),
+                    now_ts,
+                )
+                cur.execute(sql, params)
+                saved_count += 1
+        self.conn.commit()
+        return saved_count
+
+
+def _build_result(
+    *,
+    success: bool,
+    candidate_count: int,
+    saved_count: int,
+    source_url: str = "",
+    skipped_count: int = 0,
+    errors: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    errors = errors or []
+    summary_text = (
+        f"第三方赛事赔率抓取完成: 来源 {source_url or '-'}, 待写入 {candidate_count} 场, "
+        f"入库 {saved_count} 场, 跳过 {skipped_count} 项, 失败 {len(errors)} 项"
+    )
+    result = {
+        "success": bool(success),
+        "candidate_count": int(candidate_count),
+        "saved_count": int(saved_count),
+        "count": int(saved_count),
+        "source_url": source_url,
+        "skipped_count": int(skipped_count),
+        "failed_count": len(errors),
+        "summary_text": summary_text,
+    }
+    if errors:
+        result["error"] = "; ".join(errors[:5])
+    return result
+
+
+def run(db_config: Dict[str, Any], env: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
+    env = env if env is not None else os.environ
+    username = (env.get("ODDS_USERNAME") or env.get("HG_ODDS_USERNAME") or "").strip()
+    password = (env.get("ODDS_PASSWORD") or env.get("HG_ODDS_PASSWORD") or "").strip()
+    if not username or not password:
+        return _build_result(
+            success=False,
+            candidate_count=0,
+            saved_count=0,
+            errors=["ODDS_USERNAME/ODDS_PASSWORD 未配置"],
+        )
+
+    base_urls = [_normalize_base_url(url) for url in _split_env_list(env.get("ODDS_BASE_URLS", ""), DEFAULT_BASE_URLS)]
+    sport_types = [item.upper() for item in _split_env_list(env.get("ODDS_SPORT_TYPES", ""), DEFAULT_SPORT_TYPES)]
+    show_types = [item.lower() for item in _split_env_list(env.get("ODDS_SHOW_TYPES", ""), DEFAULT_SHOW_TYPES)]
+    timeout = _safe_int(env.get("ODDS_TIMEOUT"), 20)
+    verify_tls = _bool_env(env.get("ODDS_VERIFY_TLS"), default=False)
+    langx = (env.get("ODDS_LANGX") or "zh-cn").strip()
+    odds_type = (env.get("ODDS_TYPE") or "H").strip().upper()
+    request_delay = float(env.get("ODDS_REQUEST_DELAY_SECONDS") or REQUEST_DELAY_SECONDS)
+
+    client: Optional[HgOddsClient] = None
+    login_errors: List[str] = []
+    for base_url in base_urls:
+        try:
+            candidate = HgOddsClient(
+                base_url=base_url,
+                username=username,
+                password=password,
+                timeout=timeout,
+                verify_tls=verify_tls,
+                langx=langx,
+                odds_type=odds_type,
+            )
+            candidate.login()
+            client = candidate
+            break
+        except Exception as exc:  # noqa: BLE001
+            login_errors.append(f"{base_url}: {exc}")
+
+    if client is None:
+        return _build_result(success=False, candidate_count=0, saved_count=0, errors=login_errors)
+
+    items: List[Dict[str, Any]] = []
+    fetch_errors: List[str] = []
+    skipped_count = 0
+    for sport_type in sport_types:
+        for show_type in show_types:
+            try:
+                items.extend(client.fetch_game_list(sport_type, show_type))
+            except OddsGameListUnavailable:
+                skipped_count += 1
+            except Exception as exc:  # noqa: BLE001
+                fetch_errors.append(f"{sport_type}/{show_type}: {exc}")
+            if request_delay > 0:
+                time.sleep(request_delay)
+
+    repo = None
+    try:
+        repo = MatchOddsRepository(db_config)
+        repo.ensure_table()
+        saved_count = repo.upsert_latest(items, now_ts=int(time.time()))
+    except Exception as exc:  # noqa: BLE001
+        return _build_result(
+            success=False,
+            candidate_count=len(items),
+            saved_count=0,
+            source_url=client.base_url,
+            skipped_count=skipped_count,
+            errors=fetch_errors + [f"database: {exc}"],
+        )
+    finally:
+        if repo is not None:
+            repo.close()
+
+    return _build_result(
+        success=len(fetch_errors) == 0,
+        candidate_count=len(items),
+        saved_count=saved_count,
+        source_url=client.base_url,
+        skipped_count=skipped_count,
+        errors=fetch_errors,
+    )
+
+
+if __name__ == "__main__":
+    from src.core.config import load_config
+
+    cfg = load_config().database
+    print(
+        json.dumps(
+            run(
+                {
+                    "host": cfg.host,
+                    "port": cfg.port,
+                    "user": cfg.username,
+                    "password": cfg.password,
+                    "database": cfg.database,
+                    "charset": cfg.charset,
+                    "prefix": cfg.prefix,
+                }
+            ),
+            ensure_ascii=False,
+        )
+    )

docker/crawler/tests/test_match_odds.py

@@ -0,0 +1,247 @@
+import json
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+import match_odds
+from match_odds import MatchOddsRepository, OddsGameListUnavailable, parse_game_list_xml, run
+
+
+SAMPLE_GAME_LIST_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<serverresponse sip="14.39">
+  <dataCount>1</dataCount>
+  <totalDataCount>1</totalDataCount>
+  <pageCount>1</pageCount>
+  <ec id="ec11290788" hasEC="Y" myGame="ft">
+    <game id="8892295">
+      <GID>8892295</GID>
+      <DATETIME>07-09 04:00p</DATETIME>
+      <LID>104036</LID>
+      <LEAGUE>世界杯2026(美加墨)</LEAGUE>
+      <GIDM>7972917</GIDM>
+      <GNUM_H>402008</GNUM_H>
+      <GNUM_C>402007</GNUM_C>
+      <TEAM_H_ID>122821</TEAM_H_ID>
+      <TEAM_C_ID>122841</TEAM_C_ID>
+      <TEAM_H>法国</TEAM_H>
+      <TEAM_C>摩洛哥</TEAM_C>
+      <STRONG>H</STRONG>
+      <MORE>162</MORE>
+      <ECID>11290788</ECID>
+      <RATIO_R>1</RATIO_R>
+      <IOR_RH>1.030</IOR_RH>
+      <IOR_RC>0.860</IOR_RC>
+      <RATIO_OUO>2.5</RATIO_OUO>
+      <RATIO_OUU>2.5</RATIO_OUU>
+      <IOR_OUH>0.850</IOR_OUH>
+      <IOR_OUC>1.030</IOR_OUC>
+      <IOR_MH>1.57</IOR_MH>
+      <IOR_MC>6.60</IOR_MC>
+      <IOR_MN>3.90</IOR_MN>
+      <RATIO_HR>0 / 0.5</RATIO_HR>
+      <IOR_HRH>0.810</IOR_HRH>
+      <IOR_HRC>1.070</IOR_HRC>
+      <IOR_HMH>2.28</IOR_HMH>
+      <IOR_HMC>5.90</IOR_HMC>
+      <IOR_HMN>2.11</IOR_HMN>
+      <SYSTIME>2026-07-09 10:29:39</SYSTIME>
+      <IS_RB>N</IS_RB>
+    </game>
+  </ec>
+</serverresponse>
+"""
+
+
+class FakeCursor:
+    def __init__(self):
+        self.executed = []
+        self.rowcount = 1
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+    def execute(self, sql, params=None):
+        self.executed.append((sql, params))
+
+
+class FakeConnection:
+    def __init__(self):
+        self.cursor_instance = FakeCursor()
+        self.commit_count = 0
+
+    def cursor(self):
+        return self.cursor_instance
+
+    def commit(self):
+        self.commit_count += 1
+
+
+class MatchOddsTest(unittest.TestCase):
+    def _build_repo(self, prefix="la_"):
+        repo = object.__new__(MatchOddsRepository)
+        repo.prefix = prefix
+        repo.conn = FakeConnection()
+        return repo
+
+    def test_parse_game_list_xml_extracts_main_markets_and_raw_data(self):
+        items = parse_game_list_xml(
+            SAMPLE_GAME_LIST_XML,
+            source_site="hg",
+            source_url="https://mos022.com/",
+            sport_type="FT",
+            show_type="today",
+            odds_type="H",
+        )
+
+        self.assertEqual(len(items), 1)
+        item = items[0]
+        self.assertEqual(item["source_match_id"], "8892295")
+        self.assertEqual(item["league_name"], "世界杯2026(美加墨)")
+        self.assertEqual(item["home_team"], "法国")
+        self.assertEqual(item["away_team"], "摩洛哥")
+        self.assertEqual(item["match_time_text"], "07-09 04:00p")
+        self.assertEqual(item["handicap_line"], "1")
+        self.assertEqual(item["handicap_home_odds"], "1.030")
+        self.assertEqual(item["handicap_away_odds"], "0.860")
+        self.assertEqual(item["total_line"], "2.5")
+        self.assertEqual(item["total_over_odds"], "1.030")
+        self.assertEqual(item["total_under_odds"], "0.850")
+        self.assertEqual(item["home_win_odds"], "1.57")
+        self.assertEqual(item["away_win_odds"], "6.60")
+        self.assertEqual(item["draw_odds"], "3.90")
+        self.assertEqual(item["raw_data"]["GID"], "8892295")
+        self.assertEqual(item["markets"]["first_half"]["home_win"], "2.28")
+
+    def test_repository_upserts_latest_odds_by_source_match_id(self):
+        items = parse_game_list_xml(
+            SAMPLE_GAME_LIST_XML,
+            source_site="hg",
+            source_url="https://mos022.com/",
+            sport_type="FT",
+            show_type="today",
+            odds_type="H",
+        )
+        repo = self._build_repo()
+
+        saved = repo.upsert_latest(items, now_ts=1783607379)
+
+        self.assertEqual(saved, 1)
+        self.assertEqual(repo.conn.commit_count, 1)
+        sql, params = repo.conn.cursor_instance.executed[0]
+        self.assertIn("INSERT INTO `la_match_odds`", sql)
+        self.assertIn("ON DUPLICATE KEY UPDATE", sql)
+        self.assertIn("`source_match_id`", sql)
+        self.assertEqual(params[0], "hg")
+        self.assertEqual(params[2], "8892295")
+        self.assertEqual(json.loads(params[-3])["GID"], "8892295")
+        self.assertEqual(json.loads(params[-2])["moneyline"]["draw"], "3.90")
+
+    def test_run_skips_without_credentials(self):
+        result = run({"prefix": "la_"}, env={})
+
+        self.assertFalse(result["success"])
+        self.assertEqual(result["candidate_count"], 0)
+        self.assertIn("ODDS_USERNAME", result["error"])
+
+    def test_run_returns_structured_failure_when_database_is_unavailable(self):
+        class FakeClient:
+            base_url = "https://mos022.com/"
+
+            def __init__(self, **kwargs):
+                pass
+
+            def login(self):
+                return None
+
+            def fetch_game_list(self, sport_type, show_type):
+                return parse_game_list_xml(
+                    SAMPLE_GAME_LIST_XML,
+                    source_site="hg",
+                    source_url="https://mos022.com/",
+                    sport_type=sport_type,
+                    show_type=show_type,
+                    odds_type="H",
+                )
+
+        with mock.patch.object(match_odds, "HgOddsClient", FakeClient):
+            with mock.patch.object(match_odds, "MatchOddsRepository", side_effect=RuntimeError("db down")):
+                result = run(
+                    {"prefix": "la_"},
+                    env={
+                        "ODDS_USERNAME": "u",
+                        "ODDS_PASSWORD": "p",
+                        "ODDS_SPORT_TYPES": "FT",
+                        "ODDS_SHOW_TYPES": "today",
+                        "ODDS_REQUEST_DELAY_SECONDS": "0",
+                    },
+                )
+
+        self.assertFalse(result["success"])
+        self.assertEqual(result["candidate_count"], 1)
+        self.assertIn("db down", result["error"])
+
+    def test_run_skips_unavailable_game_list_combinations_without_failing(self):
+        class FakeClient:
+            base_url = "https://mos022.com/"
+
+            def __init__(self, **kwargs):
+                pass
+
+            def login(self):
+                return None
+
+            def fetch_game_list(self, sport_type, show_type):
+                if show_type == "early":
+                    raise OddsGameListUnavailable("早盘日期不可用")
+                return parse_game_list_xml(
+                    SAMPLE_GAME_LIST_XML,
+                    source_site="hg",
+                    source_url="https://mos022.com/",
+                    sport_type=sport_type,
+                    show_type=show_type,
+                    odds_type="H",
+                )
+
+        class FakeRepo:
+            def __init__(self, db_config):
+                pass
+
+            def ensure_table(self):
+                pass
+
+            def upsert_latest(self, items, now_ts=None):
+                return len(items)
+
+            def close(self):
+                pass
+
+        with mock.patch.object(match_odds, "HgOddsClient", FakeClient):
+            with mock.patch.object(match_odds, "MatchOddsRepository", FakeRepo):
+                result = run(
+                    {"prefix": "la_"},
+                    env={
+                        "ODDS_USERNAME": "u",
+                        "ODDS_PASSWORD": "p",
+                        "ODDS_SPORT_TYPES": "FT",
+                        "ODDS_SHOW_TYPES": "today,early",
+                        "ODDS_REQUEST_DELAY_SECONDS": "0",
+                    },
+                )
+
+        self.assertTrue(result["success"])
+        self.assertEqual(result["candidate_count"], 1)
+        self.assertEqual(result["saved_count"], 1)
+        self.assertEqual(result["skipped_count"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main()