432 lines
17 KiB
Python
432 lines
17 KiB
Python
import hashlib
|
||
import html
|
||
import json
|
||
import re
|
||
import time
|
||
from datetime import datetime
|
||
from urllib.parse import urljoin
|
||
|
||
import pymysql
|
||
import requests
|
||
|
||
BASE_URL = "https://cc33.49bd28.com/"
|
||
CDN_FALLBACK = "https://tk2cdn-hw.thc517.com"
|
||
IMAGE_CDN_HOST = "https://tk2cdn.ai4funs.com"
|
||
SOURCE = "49baodian"
|
||
USER_ACCOUNT = "lottery_publisher"
|
||
USER_NICKNAME = "彩票资料员"
|
||
USER_SN_START = 49000001
|
||
LOCK_WAIT_RETRY_CODES = {1205, 1213}
|
||
|
||
LOTTERY_VARIANTS = {
|
||
"a6": {
|
||
"name": "旧澳六合",
|
||
"aggregate_tag": "彩票",
|
||
"game_type": 2032,
|
||
"issue_info_url": "https://dokv.buyacard.cc/kv/gr/a6/issue/currentInfo",
|
||
"recommend_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/issue/a6/recommend",
|
||
"serial_list_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/a6/serialList",
|
||
},
|
||
"xa6": {
|
||
"name": "新澳六合",
|
||
"aggregate_tag": "彩票",
|
||
"game_type": 5,
|
||
"issue_info_url": "https://dokv.buyacard.cc/kv/gr/xa6/issue/currentInfo",
|
||
"recommend_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/issue/xa6/recommend",
|
||
"serial_list_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/xa6/serialList",
|
||
},
|
||
}
|
||
|
||
|
||
def table(prefix: str, name: str) -> str:
|
||
return f"`{prefix}{name}`"
|
||
|
||
|
||
def astro_decode(value):
|
||
if isinstance(value, list) and len(value) == 2 and isinstance(value[0], int):
|
||
value_type, payload = value
|
||
if value_type == 0:
|
||
if isinstance(payload, dict):
|
||
return {key: astro_decode(val) for key, val in payload.items()}
|
||
return payload
|
||
if value_type == 1:
|
||
return [astro_decode(item) for item in payload]
|
||
if value_type in (2, 3, 6, 7):
|
||
return payload
|
||
if value_type in (4, 5):
|
||
return [astro_decode(item) for item in payload]
|
||
if value_type in (8, 9, 10):
|
||
return payload
|
||
return payload
|
||
if isinstance(value, dict):
|
||
return {key: astro_decode(val) for key, val in value.items()}
|
||
if isinstance(value, list):
|
||
return [astro_decode(item) for item in value]
|
||
return value
|
||
|
||
|
||
def fetch_home_html(session: requests.Session) -> str:
|
||
resp = session.get(BASE_URL, timeout=30)
|
||
resp.raise_for_status()
|
||
return resp.content.decode("utf-8", errors="replace")
|
||
|
||
|
||
def extract_cdn_host(page_html: str) -> str:
|
||
match = re.search(r"window\.CDN_HOST\s*=\s*['\"]([^'\"]+)['\"]", page_html)
|
||
return match.group(1).rstrip("/") if match else CDN_FALLBACK
|
||
|
||
|
||
def extract_home_data(page_html: str) -> dict:
|
||
for match in re.finditer(r"<astro-island\b[^>]*\bprops=([\"'])(.*?)\1", page_html, re.S):
|
||
props_raw = html.unescape(match.group(2))
|
||
try:
|
||
props = astro_decode(json.loads(props_raw))
|
||
except json.JSONDecodeError:
|
||
continue
|
||
data = props.get("data") if isinstance(props, dict) else None
|
||
if isinstance(data, dict) and (data.get("lotteryResult") or data.get("galleryResult")):
|
||
return data
|
||
return {}
|
||
|
||
|
||
def request_json(session: requests.Session, url: str) -> dict:
|
||
resp = session.get(url, timeout=30)
|
||
resp.raise_for_status()
|
||
return resp.json()
|
||
|
||
|
||
WUXING_MAP = {"j": "金", "m": "木", "s": "水", "h": "火", "t": "土"}
|
||
ODD_EVEN_MAP = {"o": "单", "e": "双"}
|
||
SIZE_MAP = {"b": "大", "s": "小"}
|
||
|
||
|
||
def color_text(color: str) -> str:
|
||
return {"red": "红波", "blue": "蓝波", "green": "绿波", "R": "红波", "G": "绿波", "B": "蓝波"}.get(color, color or "")
|
||
|
||
|
||
def format_time(value) -> str:
|
||
if not value:
|
||
return ""
|
||
if isinstance(value, str) and re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$", value):
|
||
return value
|
||
try:
|
||
ts = int(value)
|
||
except (TypeError, ValueError):
|
||
return ""
|
||
if ts > 100000000000:
|
||
ts = ts // 1000
|
||
return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
def image_url(path: str, cdn_host: str) -> str:
|
||
if not path:
|
||
return ""
|
||
if path.startswith("http://") or path.startswith("https://"):
|
||
return path
|
||
normalized = path.lstrip("/")
|
||
if normalized.startswith("prod/") or normalized.startswith("prodmedia/"):
|
||
return urljoin(IMAGE_CDN_HOST + "/", normalized)
|
||
return urljoin(cdn_host.rstrip("/") + "/", normalized)
|
||
|
||
|
||
def int_value(value) -> int:
|
||
try:
|
||
return int(value)
|
||
except (TypeError, ValueError):
|
||
return 0
|
||
|
||
|
||
def is_retryable_db_error(exc: Exception) -> bool:
|
||
return isinstance(exc, pymysql.err.OperationalError) and len(exc.args) > 0 and exc.args[0] in LOCK_WAIT_RETRY_CODES
|
||
|
||
|
||
def build_draw_post(detail: dict, lottery_key: str) -> list[dict]:
|
||
if not isinstance(detail, dict):
|
||
return []
|
||
issue = detail.get("currentCompleteIssue") or detail.get("currentIssue") or detail.get("issue")
|
||
if not issue:
|
||
return []
|
||
|
||
numbers = detail.get("processedOpenCode") or detail.get("openCode") or detail.get("currentResult") or []
|
||
year = detail.get("currentYear") or detail.get("year", "")
|
||
lines = ["【49宝典开奖】", f"{year}年第{issue}期"]
|
||
|
||
for index, number in enumerate(numbers):
|
||
label = "特码" if index == len(numbers) - 1 else f"第{index + 1}球"
|
||
five_el = number.get("ws") or number.get("fiveElements") or ""
|
||
five_el = WUXING_MAP.get(five_el, five_el)
|
||
sz = number.get("size", "")
|
||
sz = SIZE_MAP.get(sz, sz)
|
||
oe = number.get("parity") or number.get("oddEven") or ""
|
||
oe = ODD_EVEN_MAP.get(oe, oe)
|
||
attrs = [
|
||
str(number.get("value") or number.get("num") or ""),
|
||
str(number.get("pet") or number.get("shengxiao") or ""),
|
||
str(five_el),
|
||
color_text(str(number.get("color", ""))),
|
||
str(oe),
|
||
str(number.get("combinedParity", "")),
|
||
str(sz),
|
||
str(number.get("tailSize", "")),
|
||
]
|
||
lines.append(f"{label}：" + " ".join([item for item in attrs if item]))
|
||
|
||
total_parity = detail.get("totalParity")
|
||
total_size = detail.get("totalSize")
|
||
if total_parity or total_size:
|
||
lines.append(f"总和：{total_parity or ''} {total_size or ''}".strip())
|
||
next_issue = detail.get("nextCompleteIssue") or detail.get("nextIssue")
|
||
if next_issue:
|
||
lines.append(f"下期：{next_issue}")
|
||
next_time = format_time(detail.get("nextOpenTime") or detail.get("nextTime"))
|
||
if next_time:
|
||
lines.append(f"下期开奖时间：{next_time}")
|
||
|
||
return [{
|
||
"origin_id": f"{SOURCE}:draw:{lottery_key}:{issue}",
|
||
"content": "\n".join(lines),
|
||
"images": [],
|
||
"like_count": 0,
|
||
"ext": {"type": "draw", "source": SOURCE, "lottery_key": lottery_key, "raw": detail},
|
||
"create_time": int(time.time()),
|
||
}]
|
||
|
||
|
||
def build_gallery_posts(items: list[dict], lottery_key: str, cdn_host: str) -> list[dict]:
|
||
if not isinstance(items, list):
|
||
return []
|
||
|
||
posts = []
|
||
for item in items:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
target_id = str(item.get("targetId") or item.get("newspaperCode") or "")
|
||
if not target_id:
|
||
raw_key = json.dumps(item, ensure_ascii=False, sort_keys=True)
|
||
target_id = hashlib.md5(raw_key.encode("utf-8")).hexdigest()
|
||
|
||
images = []
|
||
for image in item.get("img") or []:
|
||
if isinstance(image, dict):
|
||
url = image_url(str(image.get("url") or ""), cdn_host)
|
||
if url:
|
||
images.append(url)
|
||
|
||
author = item.get("author") if isinstance(item.get("author"), dict) else {}
|
||
lines = [f"【{item.get('title') or '49宝典资料'}】"]
|
||
if item.get("year") or item.get("issue"):
|
||
lines.append(f"期号：{item.get('year', '')}年第{item.get('issue', '')}期")
|
||
if item.get("serialName"):
|
||
lines.append(f"系列：{item.get('serialName')}")
|
||
|
||
posts.append({
|
||
"origin_id": f"{SOURCE}:gallery:{lottery_key}:{target_id}",
|
||
"legacy_origin_ids": [f"{SOURCE}:gallery:{target_id}"] if lottery_key == "a6" else [],
|
||
"content": "\n".join(lines),
|
||
"images": images,
|
||
"like_count": int_value(item.get("totalLikeCount")),
|
||
"ext": {"type": "gallery", "source": SOURCE, "lottery_key": lottery_key, "raw": item},
|
||
"create_time": int(time.time()),
|
||
})
|
||
return posts
|
||
|
||
|
||
def build_posts(issue_detail: dict, gallery_items: list[dict], lottery_key: str, cdn_host: str) -> list[dict]:
|
||
posts = []
|
||
posts.extend(build_draw_post(issue_detail, lottery_key))
|
||
posts.extend(build_gallery_posts(gallery_items, lottery_key, cdn_host))
|
||
return posts
|
||
|
||
|
||
def ensure_user(conn, prefix: str) -> int:
|
||
with conn.cursor() as cur:
|
||
cur.execute(f"SELECT id FROM {table(prefix, 'user')} WHERE account=%s LIMIT 1", (USER_ACCOUNT,))
|
||
row = cur.fetchone()
|
||
if row:
|
||
return int(row["id"])
|
||
|
||
cur.execute(f"SELECT MAX(sn) AS max_sn FROM {table(prefix, 'user')}")
|
||
max_sn = int((cur.fetchone() or {}).get("max_sn") or 0)
|
||
sn = max(USER_SN_START, max_sn + 1)
|
||
now = int(time.time())
|
||
cur.execute(
|
||
f"""INSERT INTO {table(prefix, 'user')}
|
||
(sn, avatar, real_name, nickname, account, password, mobile, sex, channel, is_disable,
|
||
login_ip, login_time, is_new_user, is_vip, vip_level, vip_expire_time, ai_free_count,
|
||
invite_code, inviter_id, user_money, user_points, total_recharge_amount, create_time, update_time)
|
||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||
(sn, "", "", USER_NICKNAME, USER_ACCOUNT, "", "", 0, 0, 0,
|
||
"", 0, 0, 0, 0, 0, 3, "", 0, 0, 0, 0, now, now)
|
||
)
|
||
return int(cur.lastrowid)
|
||
|
||
|
||
def ensure_tag(conn, prefix: str, tag_name: str) -> int:
|
||
with conn.cursor() as cur:
|
||
cur.execute(f"SELECT id, status FROM {table(prefix, 'community_tag')} WHERE name=%s LIMIT 1", (tag_name,))
|
||
row = cur.fetchone()
|
||
now = int(time.time())
|
||
if row:
|
||
tag_id = int(row["id"])
|
||
if int(row.get("status") or 0) != 1:
|
||
cur.execute(f"UPDATE {table(prefix, 'community_tag')} SET status=1 WHERE id=%s", (tag_id,))
|
||
return tag_id
|
||
|
||
cur.execute(
|
||
f"""INSERT INTO {table(prefix, 'community_tag')}
|
||
(name, icon, sort, post_count, is_hot, status, create_time)
|
||
VALUES (%s,%s,%s,%s,%s,%s,%s)""",
|
||
(tag_name, "", 10, 0, 1, 1, now)
|
||
)
|
||
return int(cur.lastrowid)
|
||
|
||
|
||
def save_post_images(conn, prefix: str, post_id: int, images: list[str], create_time: int):
|
||
with conn.cursor() as cur:
|
||
cur.execute(f"DELETE FROM {table(prefix, 'community_post_image')} WHERE post_id=%s", (post_id,))
|
||
for index, url in enumerate(images):
|
||
cur.execute(
|
||
f"INSERT INTO {table(prefix, 'community_post_image')} (post_id, image_url, sort, create_time) VALUES (%s,%s,%s,%s)",
|
||
(post_id, url, index, create_time)
|
||
)
|
||
|
||
|
||
def ensure_post_tag(conn, prefix: str, post_id: int, tag_id: int):
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
f"INSERT IGNORE INTO {table(prefix, 'community_post_tag')} (post_id, tag_id) VALUES (%s,%s)",
|
||
(post_id, tag_id)
|
||
)
|
||
|
||
|
||
def update_tag_count(conn, prefix: str, tag_id: int):
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
f"""UPDATE {table(prefix, 'community_tag')} SET post_count=(
|
||
SELECT COUNT(*) FROM {table(prefix, 'community_post_tag')} pt
|
||
INNER JOIN {table(prefix, 'community_post')} p ON p.id=pt.post_id
|
||
WHERE pt.tag_id=%s AND p.status=1 AND p.delete_time IS NULL
|
||
) WHERE id=%s""",
|
||
(tag_id, tag_id)
|
||
)
|
||
|
||
|
||
def sync_to_db(conn, prefix: str, posts: list[dict], user_id: int, tag_ids: list[int]) -> tuple[int, int]:
|
||
inserted = 0
|
||
updated = 0
|
||
now = int(time.time())
|
||
|
||
for post in posts:
|
||
images = post.get("images") or []
|
||
images_json = json.dumps(images, ensure_ascii=False)
|
||
ext_json = json.dumps(post.get("ext") or {}, ensure_ascii=False)
|
||
create_time = int(post.get("create_time") or now)
|
||
legacy_origin_ids = [item for item in (post.get("legacy_origin_ids") or []) if item]
|
||
|
||
post_id = 0
|
||
for attempt in range(3):
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
f"SELECT id FROM {table(prefix, 'community_post')} WHERE origin_id=%s LIMIT 1",
|
||
(post["origin_id"],)
|
||
)
|
||
existing = cur.fetchone()
|
||
if not existing and legacy_origin_ids:
|
||
placeholders = ",".join(["%s"] * len(legacy_origin_ids))
|
||
cur.execute(
|
||
f"SELECT id FROM {table(prefix, 'community_post')} WHERE origin_id IN ({placeholders}) ORDER BY id DESC LIMIT 1",
|
||
tuple(legacy_origin_ids)
|
||
)
|
||
existing = cur.fetchone()
|
||
if existing:
|
||
post_id = int(existing["id"])
|
||
cur.execute(
|
||
f"""UPDATE {table(prefix, 'community_post')}
|
||
SET origin_id=%s, user_id=%s, content=%s, images=%s, ext=%s, like_count=%s, status=1, update_time=%s
|
||
WHERE id=%s""",
|
||
(post["origin_id"], user_id, post["content"], images_json, ext_json, int_value(post.get("like_count")), now, post_id)
|
||
)
|
||
updated += 1
|
||
else:
|
||
cur.execute(
|
||
f"""INSERT INTO {table(prefix, 'community_post')}
|
||
(origin_id, user_id, content, images, post_type, is_paid, price_points, free_content_len,
|
||
like_count, ext, status, create_time, update_time)
|
||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||
(post["origin_id"], user_id, post["content"], images_json, 0, 0, 0, 100,
|
||
int_value(post.get("like_count")), ext_json, 1, create_time, now)
|
||
)
|
||
post_id = int(cur.lastrowid)
|
||
inserted += 1
|
||
break
|
||
except Exception as exc:
|
||
if not is_retryable_db_error(exc) or attempt == 2:
|
||
raise
|
||
conn.rollback()
|
||
time.sleep(0.5 * (attempt + 1))
|
||
|
||
for tag_id in tag_ids:
|
||
ensure_post_tag(conn, prefix, post_id, tag_id)
|
||
save_post_images(conn, prefix, post_id, images, create_time)
|
||
|
||
for tag_id in tag_ids:
|
||
update_tag_count(conn, prefix, tag_id)
|
||
return inserted, updated
|
||
|
||
|
||
def load_variant_payload(session: requests.Session, lottery_key: str) -> tuple[dict, list[dict]]:
|
||
variant = LOTTERY_VARIANTS[lottery_key]
|
||
issue_detail = request_json(session, variant["issue_info_url"])
|
||
gallery_payload = request_json(session, variant["recommend_url"])
|
||
gallery_items = gallery_payload.get("issueList") or []
|
||
return issue_detail, gallery_items
|
||
|
||
|
||
def run(db_config: dict):
|
||
config = dict(db_config)
|
||
prefix = config.pop("prefix", "la_")
|
||
session = requests.Session()
|
||
session.headers.update({
|
||
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Referer": BASE_URL,
|
||
})
|
||
|
||
page_html = fetch_home_html(session)
|
||
cdn_host = extract_cdn_host(page_html)
|
||
data = extract_home_data(page_html)
|
||
if not data:
|
||
raise RuntimeError("未解析到49宝典首页props数据")
|
||
|
||
conn = pymysql.connect(**config, cursorclass=pymysql.cursors.DictCursor)
|
||
try:
|
||
user_id = ensure_user(conn, prefix)
|
||
aggregate_tag_id = ensure_tag(conn, prefix, "彩票")
|
||
total_inserted = 0
|
||
total_updated = 0
|
||
total_candidates = 0
|
||
|
||
for lottery_key, variant in LOTTERY_VARIANTS.items():
|
||
issue_detail, gallery_items = load_variant_payload(session, lottery_key)
|
||
posts = build_posts(issue_detail, gallery_items, lottery_key, cdn_host)
|
||
print(f"{variant['name']} 解析到 {len(posts)} 条49宝典数据")
|
||
total_candidates += len(posts)
|
||
if not posts:
|
||
continue
|
||
|
||
tag_id = ensure_tag(conn, prefix, variant["name"])
|
||
inserted, updated = sync_to_db(conn, prefix, posts, user_id, [aggregate_tag_id, tag_id])
|
||
conn.commit()
|
||
total_inserted += inserted
|
||
total_updated += updated
|
||
|
||
summary = f"发布用户ID {user_id}, 聚合分类ID {aggregate_tag_id}, 新增 {total_inserted} 条, 更新 {total_updated} 条"
|
||
print(summary)
|
||
return {"success": True, "candidate_count": total_candidates, "saved_count": total_inserted + total_updated, "summary_text": summary}
|
||
except Exception:
|
||
conn.rollback()
|
||
raise
|
||
finally:
|
||
conn.close()
|