迁移目录
This commit is contained in:
hajimi
@@ -0,0 +1,431 @@
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import pymysql
|
||||
import requests
|
||||
|
||||
BASE_URL = "https://cc33.49bd28.com/"
|
||||
CDN_FALLBACK = "https://tk2cdn-hw.thc517.com"
|
||||
IMAGE_CDN_HOST = "https://tk2cdn.ai4funs.com"
|
||||
SOURCE = "49baodian"
|
||||
USER_ACCOUNT = "lottery_publisher"
|
||||
USER_NICKNAME = "彩票资料员"
|
||||
USER_SN_START = 49000001
|
||||
LOCK_WAIT_RETRY_CODES = {1205, 1213}
|
||||
|
||||
LOTTERY_VARIANTS = {
|
||||
"a6": {
|
||||
"name": "旧澳六合",
|
||||
"aggregate_tag": "彩票",
|
||||
"game_type": 2032,
|
||||
"issue_info_url": "https://dokv.buyacard.cc/kv/gr/a6/issue/currentInfo",
|
||||
"recommend_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/issue/a6/recommend",
|
||||
"serial_list_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/a6/serialList",
|
||||
},
|
||||
"xa6": {
|
||||
"name": "新澳六合",
|
||||
"aggregate_tag": "彩票",
|
||||
"game_type": 5,
|
||||
"issue_info_url": "https://dokv.buyacard.cc/kv/gr/xa6/issue/currentInfo",
|
||||
"recommend_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/issue/xa6/recommend",
|
||||
"serial_list_url": "https://ocs.ai4funs.com/pwtkprd/tk/pw02tk02/xa6/serialList",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def table(prefix: str, name: str) -> str:
|
||||
return f"`{prefix}{name}`"
|
||||
|
||||
|
||||
def astro_decode(value):
|
||||
if isinstance(value, list) and len(value) == 2 and isinstance(value[0], int):
|
||||
value_type, payload = value
|
||||
if value_type == 0:
|
||||
if isinstance(payload, dict):
|
||||
return {key: astro_decode(val) for key, val in payload.items()}
|
||||
return payload
|
||||
if value_type == 1:
|
||||
return [astro_decode(item) for item in payload]
|
||||
if value_type in (2, 3, 6, 7):
|
||||
return payload
|
||||
if value_type in (4, 5):
|
||||
return [astro_decode(item) for item in payload]
|
||||
if value_type in (8, 9, 10):
|
||||
return payload
|
||||
return payload
|
||||
if isinstance(value, dict):
|
||||
return {key: astro_decode(val) for key, val in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [astro_decode(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def fetch_home_html(session: requests.Session) -> str:
|
||||
resp = session.get(BASE_URL, timeout=30)
|
||||
resp.raise_for_status()
|
||||
return resp.content.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def extract_cdn_host(page_html: str) -> str:
|
||||
match = re.search(r"window\.CDN_HOST\s*=\s*['\"]([^'\"]+)['\"]", page_html)
|
||||
return match.group(1).rstrip("/") if match else CDN_FALLBACK
|
||||
|
||||
|
||||
def extract_home_data(page_html: str) -> dict:
|
||||
for match in re.finditer(r"<astro-island\b[^>]*\bprops=([\"'])(.*?)\1", page_html, re.S):
|
||||
props_raw = html.unescape(match.group(2))
|
||||
try:
|
||||
props = astro_decode(json.loads(props_raw))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
data = props.get("data") if isinstance(props, dict) else None
|
||||
if isinstance(data, dict) and (data.get("lotteryResult") or data.get("galleryResult")):
|
||||
return data
|
||||
return {}
|
||||
|
||||
|
||||
def request_json(session: requests.Session, url: str) -> dict:
|
||||
resp = session.get(url, timeout=30)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
WUXING_MAP = {"j": "金", "m": "木", "s": "水", "h": "火", "t": "土"}
|
||||
ODD_EVEN_MAP = {"o": "单", "e": "双"}
|
||||
SIZE_MAP = {"b": "大", "s": "小"}
|
||||
|
||||
|
||||
def color_text(color: str) -> str:
|
||||
return {"red": "红波", "blue": "蓝波", "green": "绿波", "R": "红波", "G": "绿波", "B": "蓝波"}.get(color, color or "")
|
||||
|
||||
|
||||
def format_time(value) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
if isinstance(value, str) and re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$", value):
|
||||
return value
|
||||
try:
|
||||
ts = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return ""
|
||||
if ts > 100000000000:
|
||||
ts = ts // 1000
|
||||
return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def image_url(path: str, cdn_host: str) -> str:
|
||||
if not path:
|
||||
return ""
|
||||
if path.startswith("http://") or path.startswith("https://"):
|
||||
return path
|
||||
normalized = path.lstrip("/")
|
||||
if normalized.startswith("prod/") or normalized.startswith("prodmedia/"):
|
||||
return urljoin(IMAGE_CDN_HOST + "/", normalized)
|
||||
return urljoin(cdn_host.rstrip("/") + "/", normalized)
|
||||
|
||||
|
||||
def int_value(value) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def is_retryable_db_error(exc: Exception) -> bool:
|
||||
return isinstance(exc, pymysql.err.OperationalError) and len(exc.args) > 0 and exc.args[0] in LOCK_WAIT_RETRY_CODES
|
||||
|
||||
|
||||
def build_draw_post(detail: dict, lottery_key: str) -> list[dict]:
|
||||
if not isinstance(detail, dict):
|
||||
return []
|
||||
issue = detail.get("currentCompleteIssue") or detail.get("currentIssue") or detail.get("issue")
|
||||
if not issue:
|
||||
return []
|
||||
|
||||
numbers = detail.get("processedOpenCode") or detail.get("openCode") or detail.get("currentResult") or []
|
||||
year = detail.get("currentYear") or detail.get("year", "")
|
||||
lines = ["【49宝典开奖】", f"{year}年第{issue}期"]
|
||||
|
||||
for index, number in enumerate(numbers):
|
||||
label = "特码" if index == len(numbers) - 1 else f"第{index + 1}球"
|
||||
five_el = number.get("ws") or number.get("fiveElements") or ""
|
||||
five_el = WUXING_MAP.get(five_el, five_el)
|
||||
sz = number.get("size", "")
|
||||
sz = SIZE_MAP.get(sz, sz)
|
||||
oe = number.get("parity") or number.get("oddEven") or ""
|
||||
oe = ODD_EVEN_MAP.get(oe, oe)
|
||||
attrs = [
|
||||
str(number.get("value") or number.get("num") or ""),
|
||||
str(number.get("pet") or number.get("shengxiao") or ""),
|
||||
str(five_el),
|
||||
color_text(str(number.get("color", ""))),
|
||||
str(oe),
|
||||
str(number.get("combinedParity", "")),
|
||||
str(sz),
|
||||
str(number.get("tailSize", "")),
|
||||
]
|
||||
lines.append(f"{label}：" + " ".join([item for item in attrs if item]))
|
||||
|
||||
total_parity = detail.get("totalParity")
|
||||
total_size = detail.get("totalSize")
|
||||
if total_parity or total_size:
|
||||
lines.append(f"总和：{total_parity or ''} {total_size or ''}".strip())
|
||||
next_issue = detail.get("nextCompleteIssue") or detail.get("nextIssue")
|
||||
if next_issue:
|
||||
lines.append(f"下期：{next_issue}")
|
||||
next_time = format_time(detail.get("nextOpenTime") or detail.get("nextTime"))
|
||||
if next_time:
|
||||
lines.append(f"下期开奖时间：{next_time}")
|
||||
|
||||
return [{
|
||||
"origin_id": f"{SOURCE}:draw:{lottery_key}:{issue}",
|
||||
"content": "\n".join(lines),
|
||||
"images": [],
|
||||
"like_count": 0,
|
||||
"ext": {"type": "draw", "source": SOURCE, "lottery_key": lottery_key, "raw": detail},
|
||||
"create_time": int(time.time()),
|
||||
}]
|
||||
|
||||
|
||||
def build_gallery_posts(items: list[dict], lottery_key: str, cdn_host: str) -> list[dict]:
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
|
||||
posts = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
target_id = str(item.get("targetId") or item.get("newspaperCode") or "")
|
||||
if not target_id:
|
||||
raw_key = json.dumps(item, ensure_ascii=False, sort_keys=True)
|
||||
target_id = hashlib.md5(raw_key.encode("utf-8")).hexdigest()
|
||||
|
||||
images = []
|
||||
for image in item.get("img") or []:
|
||||
if isinstance(image, dict):
|
||||
url = image_url(str(image.get("url") or ""), cdn_host)
|
||||
if url:
|
||||
images.append(url)
|
||||
|
||||
author = item.get("author") if isinstance(item.get("author"), dict) else {}
|
||||
lines = [f"【{item.get('title') or '49宝典资料'}】"]
|
||||
if item.get("year") or item.get("issue"):
|
||||
lines.append(f"期号：{item.get('year', '')}年第{item.get('issue', '')}期")
|
||||
if item.get("serialName"):
|
||||
lines.append(f"系列：{item.get('serialName')}")
|
||||
|
||||
posts.append({
|
||||
"origin_id": f"{SOURCE}:gallery:{lottery_key}:{target_id}",
|
||||
"legacy_origin_ids": [f"{SOURCE}:gallery:{target_id}"] if lottery_key == "a6" else [],
|
||||
"content": "\n".join(lines),
|
||||
"images": images,
|
||||
"like_count": int_value(item.get("totalLikeCount")),
|
||||
"ext": {"type": "gallery", "source": SOURCE, "lottery_key": lottery_key, "raw": item},
|
||||
"create_time": int(time.time()),
|
||||
})
|
||||
return posts
|
||||
|
||||
|
||||
def build_posts(issue_detail: dict, gallery_items: list[dict], lottery_key: str, cdn_host: str) -> list[dict]:
|
||||
posts = []
|
||||
posts.extend(build_draw_post(issue_detail, lottery_key))
|
||||
posts.extend(build_gallery_posts(gallery_items, lottery_key, cdn_host))
|
||||
return posts
|
||||
|
||||
|
||||
def ensure_user(conn, prefix: str) -> int:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"SELECT id FROM {table(prefix, 'user')} WHERE account=%s LIMIT 1", (USER_ACCOUNT,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
return int(row["id"])
|
||||
|
||||
cur.execute(f"SELECT MAX(sn) AS max_sn FROM {table(prefix, 'user')}")
|
||||
max_sn = int((cur.fetchone() or {}).get("max_sn") or 0)
|
||||
sn = max(USER_SN_START, max_sn + 1)
|
||||
now = int(time.time())
|
||||
cur.execute(
|
||||
f"""INSERT INTO {table(prefix, 'user')}
|
||||
(sn, avatar, real_name, nickname, account, password, mobile, sex, channel, is_disable,
|
||||
login_ip, login_time, is_new_user, is_vip, vip_level, vip_expire_time, ai_free_count,
|
||||
invite_code, inviter_id, user_money, user_points, total_recharge_amount, create_time, update_time)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(sn, "", "", USER_NICKNAME, USER_ACCOUNT, "", "", 0, 0, 0,
|
||||
"", 0, 0, 0, 0, 0, 3, "", 0, 0, 0, 0, now, now)
|
||||
)
|
||||
return int(cur.lastrowid)
|
||||
|
||||
|
||||
def ensure_tag(conn, prefix: str, tag_name: str) -> int:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"SELECT id, status FROM {table(prefix, 'community_tag')} WHERE name=%s LIMIT 1", (tag_name,))
|
||||
row = cur.fetchone()
|
||||
now = int(time.time())
|
||||
if row:
|
||||
tag_id = int(row["id"])
|
||||
if int(row.get("status") or 0) != 1:
|
||||
cur.execute(f"UPDATE {table(prefix, 'community_tag')} SET status=1 WHERE id=%s", (tag_id,))
|
||||
return tag_id
|
||||
|
||||
cur.execute(
|
||||
f"""INSERT INTO {table(prefix, 'community_tag')}
|
||||
(name, icon, sort, post_count, is_hot, status, create_time)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(tag_name, "", 10, 0, 1, 1, now)
|
||||
)
|
||||
return int(cur.lastrowid)
|
||||
|
||||
|
||||
def save_post_images(conn, prefix: str, post_id: int, images: list[str], create_time: int):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"DELETE FROM {table(prefix, 'community_post_image')} WHERE post_id=%s", (post_id,))
|
||||
for index, url in enumerate(images):
|
||||
cur.execute(
|
||||
f"INSERT INTO {table(prefix, 'community_post_image')} (post_id, image_url, sort, create_time) VALUES (%s,%s,%s,%s)",
|
||||
(post_id, url, index, create_time)
|
||||
)
|
||||
|
||||
|
||||
def ensure_post_tag(conn, prefix: str, post_id: int, tag_id: int):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"INSERT IGNORE INTO {table(prefix, 'community_post_tag')} (post_id, tag_id) VALUES (%s,%s)",
|
||||
(post_id, tag_id)
|
||||
)
|
||||
|
||||
|
||||
def update_tag_count(conn, prefix: str, tag_id: int):
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"""UPDATE {table(prefix, 'community_tag')} SET post_count=(
|
||||
SELECT COUNT(*) FROM {table(prefix, 'community_post_tag')} pt
|
||||
INNER JOIN {table(prefix, 'community_post')} p ON p.id=pt.post_id
|
||||
WHERE pt.tag_id=%s AND p.status=1 AND p.delete_time IS NULL
|
||||
) WHERE id=%s""",
|
||||
(tag_id, tag_id)
|
||||
)
|
||||
|
||||
|
||||
def sync_to_db(conn, prefix: str, posts: list[dict], user_id: int, tag_ids: list[int]) -> tuple[int, int]:
|
||||
inserted = 0
|
||||
updated = 0
|
||||
now = int(time.time())
|
||||
|
||||
for post in posts:
|
||||
images = post.get("images") or []
|
||||
images_json = json.dumps(images, ensure_ascii=False)
|
||||
ext_json = json.dumps(post.get("ext") or {}, ensure_ascii=False)
|
||||
create_time = int(post.get("create_time") or now)
|
||||
legacy_origin_ids = [item for item in (post.get("legacy_origin_ids") or []) if item]
|
||||
|
||||
post_id = 0
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
f"SELECT id FROM {table(prefix, 'community_post')} WHERE origin_id=%s LIMIT 1",
|
||||
(post["origin_id"],)
|
||||
)
|
||||
existing = cur.fetchone()
|
||||
if not existing and legacy_origin_ids:
|
||||
placeholders = ",".join(["%s"] * len(legacy_origin_ids))
|
||||
cur.execute(
|
||||
f"SELECT id FROM {table(prefix, 'community_post')} WHERE origin_id IN ({placeholders}) ORDER BY id DESC LIMIT 1",
|
||||
tuple(legacy_origin_ids)
|
||||
)
|
||||
existing = cur.fetchone()
|
||||
if existing:
|
||||
post_id = int(existing["id"])
|
||||
cur.execute(
|
||||
f"""UPDATE {table(prefix, 'community_post')}
|
||||
SET origin_id=%s, user_id=%s, content=%s, images=%s, ext=%s, like_count=%s, status=1, update_time=%s
|
||||
WHERE id=%s""",
|
||||
(post["origin_id"], user_id, post["content"], images_json, ext_json, int_value(post.get("like_count")), now, post_id)
|
||||
)
|
||||
updated += 1
|
||||
else:
|
||||
cur.execute(
|
||||
f"""INSERT INTO {table(prefix, 'community_post')}
|
||||
(origin_id, user_id, content, images, post_type, is_paid, price_points, free_content_len,
|
||||
like_count, ext, status, create_time, update_time)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(post["origin_id"], user_id, post["content"], images_json, 0, 0, 0, 100,
|
||||
int_value(post.get("like_count")), ext_json, 1, create_time, now)
|
||||
)
|
||||
post_id = int(cur.lastrowid)
|
||||
inserted += 1
|
||||
break
|
||||
except Exception as exc:
|
||||
if not is_retryable_db_error(exc) or attempt == 2:
|
||||
raise
|
||||
conn.rollback()
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
|
||||
for tag_id in tag_ids:
|
||||
ensure_post_tag(conn, prefix, post_id, tag_id)
|
||||
save_post_images(conn, prefix, post_id, images, create_time)
|
||||
|
||||
for tag_id in tag_ids:
|
||||
update_tag_count(conn, prefix, tag_id)
|
||||
return inserted, updated
|
||||
|
||||
|
||||
def load_variant_payload(session: requests.Session, lottery_key: str) -> tuple[dict, list[dict]]:
|
||||
variant = LOTTERY_VARIANTS[lottery_key]
|
||||
issue_detail = request_json(session, variant["issue_info_url"])
|
||||
gallery_payload = request_json(session, variant["recommend_url"])
|
||||
gallery_items = gallery_payload.get("issueList") or []
|
||||
return issue_detail, gallery_items
|
||||
|
||||
|
||||
def run(db_config: dict):
|
||||
config = dict(db_config)
|
||||
prefix = config.pop("prefix", "la_")
|
||||
session = requests.Session()
|
||||
session.headers.update({
|
||||
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Referer": BASE_URL,
|
||||
})
|
||||
|
||||
page_html = fetch_home_html(session)
|
||||
cdn_host = extract_cdn_host(page_html)
|
||||
data = extract_home_data(page_html)
|
||||
if not data:
|
||||
raise RuntimeError("未解析到49宝典首页props数据")
|
||||
|
||||
conn = pymysql.connect(**config, cursorclass=pymysql.cursors.DictCursor)
|
||||
try:
|
||||
user_id = ensure_user(conn, prefix)
|
||||
aggregate_tag_id = ensure_tag(conn, prefix, "彩票")
|
||||
total_inserted = 0
|
||||
total_updated = 0
|
||||
total_candidates = 0
|
||||
|
||||
for lottery_key, variant in LOTTERY_VARIANTS.items():
|
||||
issue_detail, gallery_items = load_variant_payload(session, lottery_key)
|
||||
posts = build_posts(issue_detail, gallery_items, lottery_key, cdn_host)
|
||||
print(f"{variant['name']} 解析到 {len(posts)} 条49宝典数据")
|
||||
total_candidates += len(posts)
|
||||
if not posts:
|
||||
continue
|
||||
|
||||
tag_id = ensure_tag(conn, prefix, variant["name"])
|
||||
inserted, updated = sync_to_db(conn, prefix, posts, user_id, [aggregate_tag_id, tag_id])
|
||||
conn.commit()
|
||||
total_inserted += inserted
|
||||
total_updated += updated
|
||||
|
||||
summary = f"发布用户ID {user_id}, 聚合分类ID {aggregate_tag_id}, 新增 {total_inserted} 条, 更新 {total_updated} 条"
|
||||
print(summary)
|
||||
return {"success": True, "candidate_count": total_candidates, "saved_count": total_inserted + total_updated, "summary_text": summary}
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user