import hashlib import html import json import re import time from datetime import datetime from urllib.parse import urljoin import pymysql import requests BASE_URL = "https://cc33.49bd28.com/" CDN_FALLBACK = "https://tk2cdn-hw.thc517.com" IMAGE_CDN_HOST = "https://tk2cdn.ai4funs.com" SOURCE = "49baodian" USER_ACCOUNT = "lottery_publisher" USER_NICKNAME = "彩票资料员" USER_SN_START = 49000001 LOCK_WAIT_RETRY_CODES = {1205, 1213} LOTTERY_VARIANTS = { "a6": { "name": "旧澳六合", "aggregate_tag": "彩票", "game_type": 2032, "issue_info_url": "https://dokv.buyacard.cc/kv/gr/a6/issue/currentInfo", "recommend_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/issue/a6/recommend", "serial_list_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/a6/serialList", }, "xa6": { "name": "新澳六合", "aggregate_tag": "彩票", "game_type": 5, "issue_info_url": "https://dokv.buyacard.cc/kv/gr/xa6/issue/currentInfo", "recommend_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/issue/xa6/recommend", "serial_list_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/xa6/serialList", }, } def table(prefix: str, name: str) -> str: return f"`{prefix}{name}`" def astro_decode(value): if isinstance(value, list) and len(value) == 2 and isinstance(value[0], int): value_type, payload = value if value_type == 0: if isinstance(payload, dict): return {key: astro_decode(val) for key, val in payload.items()} return payload if value_type == 1: return [astro_decode(item) for item in payload] if value_type in (2, 3, 6, 7): return payload if value_type in (4, 5): return [astro_decode(item) for item in payload] if value_type in (8, 9, 10): return payload return payload if isinstance(value, dict): return {key: astro_decode(val) for key, val in value.items()} if isinstance(value, list): return [astro_decode(item) for item in value] return value def fetch_home_html(session: requests.Session) -> str: resp = session.get(BASE_URL, timeout=30) resp.raise_for_status() return resp.content.decode("utf-8", errors="replace") def extract_cdn_host(page_html: str) -> str: match = re.search(r"window\.CDN_HOST\s*=\s*['\"]([^'\"]+)['\"]", page_html) return match.group(1).rstrip("/") if match else CDN_FALLBACK def extract_home_data(page_html: str) -> dict: for match in re.finditer(r"]*\bprops=([\"'])(.*?)\1", page_html, re.S): props_raw = html.unescape(match.group(2)) try: props = astro_decode(json.loads(props_raw)) except json.JSONDecodeError: continue data = props.get("data") if isinstance(props, dict) else None if isinstance(data, dict) and (data.get("lotteryResult") or data.get("galleryResult")): return data return {} def request_json(session: requests.Session, url: str) -> dict: resp = session.get(url, timeout=30) resp.raise_for_status() return resp.json() WUXING_MAP = {"j": "金", "m": "木", "s": "水", "h": "火", "t": "土"} ODD_EVEN_MAP = {"o": "单", "e": "双"} SIZE_MAP = {"b": "大", "s": "小"} def color_text(color: str) -> str: return {"red": "红波", "blue": "蓝波", "green": "绿波", "R": "红波", "G": "绿波", "B": "蓝波"}.get(color, color or "") def format_time(value) -> str: if not value: return "" if isinstance(value, str) and re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$", value): return value try: ts = int(value) except (TypeError, ValueError): return "" if ts > 100000000000: ts = ts // 1000 return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") def image_url(path: str, cdn_host: str) -> str: if not path: return "" if path.startswith("http://") or path.startswith("https://"): return path normalized = path.lstrip("/") if normalized.startswith("prod/") or normalized.startswith("prodmedia/"): return urljoin(IMAGE_CDN_HOST + "/", normalized) return urljoin(cdn_host.rstrip("/") + "/", normalized) def int_value(value) -> int: try: return int(value) except (TypeError, ValueError): return 0 def is_retryable_db_error(exc: Exception) -> bool: return isinstance(exc, pymysql.err.OperationalError) and len(exc.args) > 0 and exc.args[0] in LOCK_WAIT_RETRY_CODES def build_draw_post(detail: dict, lottery_key: str) -> list[dict]: if not isinstance(detail, dict): return [] issue = detail.get("currentCompleteIssue") or detail.get("currentIssue") or detail.get("issue") if not issue: return [] numbers = detail.get("processedOpenCode") or detail.get("openCode") or detail.get("currentResult") or [] year = detail.get("currentYear") or detail.get("year", "") lines = ["【49宝典开奖】", f"{year}年第{issue}期"] for index, number in enumerate(numbers): label = "特码" if index == len(numbers) - 1 else f"第{index + 1}球" five_el = number.get("ws") or number.get("fiveElements") or "" five_el = WUXING_MAP.get(five_el, five_el) sz = number.get("size", "") sz = SIZE_MAP.get(sz, sz) oe = number.get("parity") or number.get("oddEven") or "" oe = ODD_EVEN_MAP.get(oe, oe) attrs = [ str(number.get("value") or number.get("num") or ""), str(number.get("pet") or number.get("shengxiao") or ""), str(five_el), color_text(str(number.get("color", ""))), str(oe), str(number.get("combinedParity", "")), str(sz), str(number.get("tailSize", "")), ] lines.append(f"{label}:" + " ".join([item for item in attrs if item])) total_parity = detail.get("totalParity") total_size = detail.get("totalSize") if total_parity or total_size: lines.append(f"总和:{total_parity or ''} {total_size or ''}".strip()) next_issue = detail.get("nextCompleteIssue") or detail.get("nextIssue") if next_issue: lines.append(f"下期:{next_issue}") next_time = format_time(detail.get("nextOpenTime") or detail.get("nextTime")) if next_time: lines.append(f"下期开奖时间:{next_time}") return [{ "origin_id": f"{SOURCE}:draw:{lottery_key}:{issue}", "content": "\n".join(lines), "images": [], "like_count": 0, "ext": {"type": "draw", "source": SOURCE, "lottery_key": lottery_key, "raw": detail}, "create_time": int(time.time()), }] def build_gallery_posts(items: list[dict], lottery_key: str, cdn_host: str) -> list[dict]: if not isinstance(items, list): return [] posts = [] for item in items: if not isinstance(item, dict): continue target_id = str(item.get("targetId") or item.get("newspaperCode") or "") if not target_id: raw_key = json.dumps(item, ensure_ascii=False, sort_keys=True) target_id = hashlib.md5(raw_key.encode("utf-8")).hexdigest() images = [] for image in item.get("img") or []: if isinstance(image, dict): url = image_url(str(image.get("url") or ""), cdn_host) if url: images.append(url) author = item.get("author") if isinstance(item.get("author"), dict) else {} lines = [f"【{item.get('title') or '49宝典资料'}】"] if item.get("year") or item.get("issue"): lines.append(f"期号:{item.get('year', '')}年第{item.get('issue', '')}期") if item.get("serialName"): lines.append(f"系列:{item.get('serialName')}") posts.append({ "origin_id": f"{SOURCE}:gallery:{lottery_key}:{target_id}", "legacy_origin_ids": [f"{SOURCE}:gallery:{target_id}"] if lottery_key == "a6" else [], "content": "\n".join(lines), "images": images, "like_count": int_value(item.get("totalLikeCount")), "ext": {"type": "gallery", "source": SOURCE, "lottery_key": lottery_key, "raw": item}, "create_time": int(time.time()), }) return posts def build_posts(issue_detail: dict, gallery_items: list[dict], lottery_key: str, cdn_host: str) -> list[dict]: posts = [] posts.extend(build_draw_post(issue_detail, lottery_key)) posts.extend(build_gallery_posts(gallery_items, lottery_key, cdn_host)) return posts def ensure_user(conn, prefix: str) -> int: with conn.cursor() as cur: cur.execute(f"SELECT id FROM {table(prefix, 'user')} WHERE account=%s LIMIT 1", (USER_ACCOUNT,)) row = cur.fetchone() if row: return int(row["id"]) cur.execute(f"SELECT MAX(sn) AS max_sn FROM {table(prefix, 'user')}") max_sn = int((cur.fetchone() or {}).get("max_sn") or 0) sn = max(USER_SN_START, max_sn + 1) now = int(time.time()) cur.execute( f"""INSERT INTO {table(prefix, 'user')} (sn, avatar, real_name, nickname, account, password, mobile, sex, channel, is_disable, login_ip, login_time, is_new_user, is_vip, vip_level, vip_expire_time, ai_free_count, invite_code, inviter_id, user_money, user_points, total_recharge_amount, create_time, update_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""", (sn, "", "", USER_NICKNAME, USER_ACCOUNT, "", "", 0, 0, 0, "", 0, 0, 0, 0, 0, 3, "", 0, 0, 0, 0, now, now) ) return int(cur.lastrowid) def ensure_tag(conn, prefix: str, tag_name: str) -> int: with conn.cursor() as cur: cur.execute(f"SELECT id, status FROM {table(prefix, 'community_tag')} WHERE name=%s LIMIT 1", (tag_name,)) row = cur.fetchone() now = int(time.time()) if row: tag_id = int(row["id"]) if int(row.get("status") or 0) != 1: cur.execute(f"UPDATE {table(prefix, 'community_tag')} SET status=1 WHERE id=%s", (tag_id,)) return tag_id cur.execute( f"""INSERT INTO {table(prefix, 'community_tag')} (name, icon, sort, post_count, is_hot, status, create_time) VALUES (%s,%s,%s,%s,%s,%s,%s)""", (tag_name, "", 10, 0, 1, 1, now) ) return int(cur.lastrowid) def save_post_images(conn, prefix: str, post_id: int, images: list[str], create_time: int): with conn.cursor() as cur: cur.execute(f"DELETE FROM {table(prefix, 'community_post_image')} WHERE post_id=%s", (post_id,)) for index, url in enumerate(images): cur.execute( f"INSERT INTO {table(prefix, 'community_post_image')} (post_id, image_url, sort, create_time) VALUES (%s,%s,%s,%s)", (post_id, url, index, create_time) ) def ensure_post_tag(conn, prefix: str, post_id: int, tag_id: int): with conn.cursor() as cur: cur.execute( f"INSERT IGNORE INTO {table(prefix, 'community_post_tag')} (post_id, tag_id) VALUES (%s,%s)", (post_id, tag_id) ) def update_tag_count(conn, prefix: str, tag_id: int): with conn.cursor() as cur: cur.execute( f"""UPDATE {table(prefix, 'community_tag')} SET post_count=( SELECT COUNT(*) FROM {table(prefix, 'community_post_tag')} pt INNER JOIN {table(prefix, 'community_post')} p ON p.id=pt.post_id WHERE pt.tag_id=%s AND p.status=1 AND p.delete_time IS NULL ) WHERE id=%s""", (tag_id, tag_id) ) def sync_to_db(conn, prefix: str, posts: list[dict], user_id: int, tag_ids: list[int]) -> tuple[int, int]: inserted = 0 updated = 0 now = int(time.time()) for post in posts: images = post.get("images") or [] images_json = json.dumps(images, ensure_ascii=False) ext_json = json.dumps(post.get("ext") or {}, ensure_ascii=False) create_time = int(post.get("create_time") or now) legacy_origin_ids = [item for item in (post.get("legacy_origin_ids") or []) if item] post_id = 0 for attempt in range(3): try: with conn.cursor() as cur: cur.execute( f"SELECT id FROM {table(prefix, 'community_post')} WHERE origin_id=%s LIMIT 1", (post["origin_id"],) ) existing = cur.fetchone() if not existing and legacy_origin_ids: placeholders = ",".join(["%s"] * len(legacy_origin_ids)) cur.execute( f"SELECT id FROM {table(prefix, 'community_post')} WHERE origin_id IN ({placeholders}) ORDER BY id DESC LIMIT 1", tuple(legacy_origin_ids) ) existing = cur.fetchone() if existing: post_id = int(existing["id"]) cur.execute( f"""UPDATE {table(prefix, 'community_post')} SET origin_id=%s, user_id=%s, content=%s, images=%s, ext=%s, like_count=%s, status=1, update_time=%s WHERE id=%s""", (post["origin_id"], user_id, post["content"], images_json, ext_json, int_value(post.get("like_count")), now, post_id) ) updated += 1 else: cur.execute( f"""INSERT INTO {table(prefix, 'community_post')} (origin_id, user_id, content, images, post_type, is_paid, price_points, free_content_len, like_count, ext, status, create_time, update_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""", (post["origin_id"], user_id, post["content"], images_json, 0, 0, 0, 100, int_value(post.get("like_count")), ext_json, 1, create_time, now) ) post_id = int(cur.lastrowid) inserted += 1 break except Exception as exc: if not is_retryable_db_error(exc) or attempt == 2: raise conn.rollback() time.sleep(0.5 * (attempt + 1)) for tag_id in tag_ids: ensure_post_tag(conn, prefix, post_id, tag_id) save_post_images(conn, prefix, post_id, images, create_time) for tag_id in tag_ids: update_tag_count(conn, prefix, tag_id) return inserted, updated def load_variant_payload(session: requests.Session, lottery_key: str) -> tuple[dict, list[dict]]: variant = LOTTERY_VARIANTS[lottery_key] issue_detail = request_json(session, variant["issue_info_url"]) gallery_payload = request_json(session, variant["recommend_url"]) gallery_items = gallery_payload.get("issueList") or [] return issue_detail, gallery_items def run(db_config: dict): config = dict(db_config) prefix = config.pop("prefix", "la_") session = requests.Session() session.headers.update({ "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Referer": BASE_URL, }) page_html = fetch_home_html(session) cdn_host = extract_cdn_host(page_html) data = extract_home_data(page_html) if not data: raise RuntimeError("未解析到49宝典首页props数据") conn = pymysql.connect(**config, cursorclass=pymysql.cursors.DictCursor) try: user_id = ensure_user(conn, prefix) aggregate_tag_id = ensure_tag(conn, prefix, "彩票") total_inserted = 0 total_updated = 0 total_candidates = 0 for lottery_key, variant in LOTTERY_VARIANTS.items(): issue_detail, gallery_items = load_variant_payload(session, lottery_key) posts = build_posts(issue_detail, gallery_items, lottery_key, cdn_host) print(f"{variant['name']} 解析到 {len(posts)} 条49宝典数据") total_candidates += len(posts) if not posts: continue tag_id = ensure_tag(conn, prefix, variant["name"]) inserted, updated = sync_to_db(conn, prefix, posts, user_id, [aggregate_tag_id, tag_id]) conn.commit() total_inserted += inserted total_updated += updated summary = f"发布用户ID {user_id}, 聚合分类ID {aggregate_tag_id}, 新增 {total_inserted} 条, 更新 {total_updated} 条" print(summary) return {"success": True, "candidate_count": total_candidates, "saved_count": total_inserted + total_updated, "summary_text": summary} except Exception: conn.rollback() raise finally: conn.close()