Files
sbnews/docker/crawler/ai_comment_dispatch.py
T

1047 lines
46 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
from __future__ import annotations
import concurrent.futures
import hashlib
import html
import json
import random
import re
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Tuple
import pymysql
import requests
DEFAULT_OPENAI_BASE_URL = "https://lingsuan.top"
DEFAULT_OPENAI_MODEL = "gpt-5.5"
DEFAULT_OPENAI_WIRE_API = "responses"
DEFAULT_OPENAI_PROVIDER_LABEL = "OpenAI"
DEFAULT_OPENAI_REASONING_EFFORT = "xhigh"
DEFAULT_OPENAI_DISABLE_RESPONSE_STORAGE = "1"
DEFAULT_REQUEST_TIMEOUT = 180
DEFAULT_RETRY_TIMES = 2
DEFAULT_RETRY_SLEEP_MS = 800
BOT_ACCOUNT_PREFIX = "ai_comment_"
BOT_HANDLE_PREFIXES = [
"看台",
"深夜",
"边线",
"北区",
"南看台",
"东门",
"西街",
"慢热",
"小酒馆",
"补时",
"角旗",
"传中",
"临场",
"半场",
"球门后",
"旧球鞋",
"赛点",
"风向",
"追球",
"回防",
]
BOT_HANDLE_SUFFIXES = [
"阿泽",
"小顾",
"老韩",
"阿凯",
"小唐",
"阿林",
"阿松",
"阿北",
"阿岳",
"阿诚",
]
DEFAULT_ARTICLE_PROMPT = (
"你是一名专业、克制、真实的体育资讯评论员。请根据给定资讯内容生成1条中文评论,"
"要求自然、具体、像真实用户发言,评论长度控制在20到60字之间。不要输出标题、前缀、解释、表情、标签、项目符号,"
"不要提及自己是AI,不要复述全文。"
)
DEFAULT_POST_PROMPT = (
"你是一名活跃在体育与彩票社区的真实用户。请根据给定帖子内容、标签和人物设定,生成1条中文评论。"
"评论要像真实社区回复,长度控制在18到66字之间,允许带轻微情绪和立场,但不要辱骂、不要承诺收益、不要暴露自己是AI。"
"只输出评论正文,不要加前缀、编号、表情或话题标签。"
)
PERSONA_PROFILES = [
{"key": "data_analyst", "label": "数据派", "role": "数据分析师", "tone": "冷静克制", "scene": "赛后复盘群", "focus": "数字与趋势"},
{"key": "old_fan", "label": "老球迷", "role": "资深老球迷", "tone": "接地气", "scene": "比赛直播间", "focus": "临场感觉"},
{"key": "coach_view", "label": "教练视角", "role": "青训教练", "tone": "专业直接", "scene": "战术讨论区", "focus": "执行与细节"},
{"key": "night_shift_editor", "label": "夜班编辑", "role": "夜班体育编辑", "tone": "简练犀利", "scene": "资讯值班台", "focus": "新闻点与影响"},
{"key": "injury_tracker", "label": "伤停控", "role": "伤停追踪爱好者", "tone": "谨慎务实", "scene": "赛前情报串", "focus": "伤病与轮换"},
{"key": "odds_reader", "label": "赔率派", "role": "盘口观察者", "tone": "稳健", "scene": "分析贴评论区", "focus": "市场变化"},
{"key": "locker_room", "label": "更衣室派", "role": "更衣室故事控", "tone": "代入感强", "scene": "球迷论坛", "focus": "情绪与气场"},
{"key": "youth_scout", "label": "球探派", "role": "青年球探", "tone": "敏锐客观", "scene": "新人观察帖", "focus": "成长潜力"},
{"key": "match_host", "label": "解说腔", "role": "民间解说员", "tone": "有画面感", "scene": "比赛串场群", "focus": "比赛走势"},
{"key": "local_supporter", "label": "主队党", "role": "本地队死忠", "tone": "有立场但不过火", "scene": "主队话题区", "focus": "球队情绪"},
{"key": "lottery_modeler", "label": "模型党", "role": "彩票模型玩家", "tone": "谨慎理性", "scene": "资料分析区", "focus": "冷热与结构"},
{"key": "news_watcher", "label": "资讯控", "role": "新闻跟踪者", "tone": "平和", "scene": "快讯评论区", "focus": "后续影响"},
]
def _truthy(value: Any) -> bool:
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
def _safe_int(value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _safe_float(value: Any, default: float) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
def normalize_text(value: str, limit: int = 800) -> str:
text = html.unescape(re.sub(r"<[^>]+>", " ", value or ""))
text = re.sub(r"\s+", " ", text).strip()
return text[:limit]
def extract_post_text(content: str, ext: Optional[Dict[str, Any]] = None) -> str:
payload = ext or {}
if payload.get("lottery_analysis_source") == "image_only":
analyzed = normalize_text(str(payload.get("lottery_analysis_content") or ""), limit=1000)
if analyzed:
return analyzed
translated = normalize_text(str(payload.get("translated_content") or ""), limit=1000)
if translated:
return translated
return normalize_text(content or "", limit=1000)
@dataclass
class CommentCandidate:
target_type: str
target_id: int
title: str
body: str
selection_sources: Tuple[str, ...]
created_at: int = 0
rank_value: int = 0
context_tags: Tuple[str, ...] = ()
metadata: Dict[str, Any] = field(default_factory=dict)
@dataclass
class VirtualUser:
user_id: int
account: str
nickname: str
persona_key: str
@dataclass
class DispatchTask:
task_id: int
target_type: str
target_id: int
virtual_user_id: int
persona_key: str
selection_sources: Tuple[str, ...]
@dataclass
class AiCommentSettings:
enabled: bool = True
user_pool_size: int = 100
per_run: int = 50
daily_cap: int = 500
concurrency: int = 4
request_timeout: int = DEFAULT_REQUEST_TIMEOUT
article_prompt: str = DEFAULT_ARTICLE_PROMPT
post_prompt: str = DEFAULT_POST_PROMPT
openai_api_key: str = ""
openai_base_url: str = DEFAULT_OPENAI_BASE_URL
openai_model: str = DEFAULT_OPENAI_MODEL
openai_wire_api: str = DEFAULT_OPENAI_WIRE_API
openai_provider_label: str = DEFAULT_OPENAI_PROVIDER_LABEL
openai_reasoning_effort: str = DEFAULT_OPENAI_REASONING_EFFORT
openai_disable_response_storage: str = DEFAULT_OPENAI_DISABLE_RESPONSE_STORAGE
max_tokens: int = 220
temperature: float = 0.75
retry_times: int = DEFAULT_RETRY_TIMES
retry_sleep_ms: int = DEFAULT_RETRY_SLEEP_MS
@classmethod
def from_config_map(cls, config: Dict[str, Any]) -> "AiCommentSettings":
article_prompt = str(
config.get("auto_comment_article_prompt")
or config.get("article_comment_prompt")
or DEFAULT_ARTICLE_PROMPT
).strip()
post_prompt = str(config.get("auto_comment_post_prompt") or DEFAULT_POST_PROMPT).strip()
return cls(
enabled=_truthy(config.get("auto_comment_enabled", "1")),
user_pool_size=max(1, _safe_int(config.get("auto_comment_user_pool_size"), 100)),
per_run=max(1, _safe_int(config.get("auto_comment_per_run"), 50)),
daily_cap=max(1, _safe_int(config.get("auto_comment_daily_cap"), 500)),
concurrency=max(1, min(10, _safe_int(config.get("auto_comment_concurrency"), 4))),
request_timeout=max(30, min(300, _safe_int(config.get("auto_comment_request_timeout"), DEFAULT_REQUEST_TIMEOUT))),
article_prompt=article_prompt,
post_prompt=post_prompt,
openai_api_key=str(config.get("openai_api_key") or "").strip(),
openai_base_url=str(config.get("openai_base_url") or DEFAULT_OPENAI_BASE_URL).strip(),
openai_model=str(config.get("openai_model") or DEFAULT_OPENAI_MODEL).strip(),
openai_wire_api=str(config.get("openai_wire_api") or DEFAULT_OPENAI_WIRE_API).strip().lower(),
openai_provider_label=str(config.get("openai_provider_label") or DEFAULT_OPENAI_PROVIDER_LABEL).strip(),
openai_reasoning_effort=str(config.get("openai_reasoning_effort") or DEFAULT_OPENAI_REASONING_EFFORT).strip(),
openai_disable_response_storage=str(
config.get("openai_disable_response_storage") or DEFAULT_OPENAI_DISABLE_RESPONSE_STORAGE
).strip(),
max_tokens=max(100, min(800, _safe_int(config.get("max_tokens"), 220))),
temperature=max(0.1, min(1.2, _safe_float(config.get("temperature"), 0.75))),
retry_times=max(0, min(3, _safe_int(config.get("ai_request_retry_times"), DEFAULT_RETRY_TIMES))),
retry_sleep_ms=max(0, min(5000, _safe_int(config.get("ai_request_retry_sleep_ms"), DEFAULT_RETRY_SLEEP_MS))),
)
def merge_candidates(candidates: Sequence[CommentCandidate]) -> List[CommentCandidate]:
merged: Dict[Tuple[str, int], CommentCandidate] = {}
order: List[Tuple[str, int]] = []
for candidate in candidates:
key = (candidate.target_type, candidate.target_id)
if key not in merged:
merged[key] = candidate
order.append(key)
continue
existing = merged[key]
merged_sources = list(existing.selection_sources)
for source in candidate.selection_sources:
if source not in merged_sources:
merged_sources.append(source)
merged_tags = list(existing.context_tags)
for tag in candidate.context_tags:
if tag not in merged_tags:
merged_tags.append(tag)
metadata = dict(existing.metadata)
for meta_key, meta_value in candidate.metadata.items():
if meta_key == "category_names":
existing_names = list(metadata.get(meta_key, ()))
for name in meta_value:
if name not in existing_names:
existing_names.append(name)
metadata[meta_key] = tuple(existing_names)
else:
metadata.setdefault(meta_key, meta_value)
merged[key] = CommentCandidate(
target_type=existing.target_type,
target_id=existing.target_id,
title=existing.title or candidate.title,
body=existing.body or candidate.body,
selection_sources=tuple(merged_sources),
created_at=max(existing.created_at, candidate.created_at),
rank_value=max(existing.rank_value, candidate.rank_value),
context_tags=tuple(merged_tags),
metadata=metadata,
)
return [merged[key] for key in order]
def select_balanced_candidates(
article_candidates: Sequence[CommentCandidate],
post_candidates: Sequence[CommentCandidate],
total_limit: int,
) -> List[CommentCandidate]:
total_limit = max(0, int(total_limit))
if total_limit == 0:
return []
article_quota = total_limit // 2
post_quota = total_limit - article_quota
selected_articles = list(article_candidates[:article_quota])
selected_posts = list(post_candidates[:post_quota])
remaining = total_limit - len(selected_articles) - len(selected_posts)
if remaining > 0:
selected_articles.extend(article_candidates[len(selected_articles): len(selected_articles) + remaining])
remaining = total_limit - len(selected_articles) - len(selected_posts)
if remaining > 0:
selected_posts.extend(post_candidates[len(selected_posts): len(selected_posts) + remaining])
return selected_articles + selected_posts
class AiCommentLLMClient:
def generate_comment(
self,
candidate: CommentCandidate,
virtual_user: VirtualUser,
persona_key: str,
settings: AiCommentSettings,
) -> Dict[str, Any]:
persona = self._persona_by_key(persona_key)
system_prompt = settings.article_prompt if candidate.target_type == "article" else settings.post_prompt
user_message = self._build_user_message(candidate, virtual_user, persona)
return self._chat(system_prompt, user_message, settings)
def _chat(self, system_prompt: str, user_message: str, settings: AiCommentSettings) -> Dict[str, Any]:
if not settings.openai_api_key:
return {"success": False, "error": f"{settings.openai_provider_label} API Key未配置"}
endpoint = self._build_endpoint(settings.openai_base_url, settings.openai_wire_api)
payload = self._build_payload(system_prompt, user_message, settings)
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {settings.openai_api_key}",
}
start = time.monotonic()
last_error = "未知错误"
for attempt in range(settings.retry_times + 1):
try:
response = requests.post(
endpoint,
headers=headers,
json=payload,
timeout=settings.request_timeout,
)
data = response.json()
except Exception as exc: # noqa: BLE001
last_error = f"请求失败: {exc}"
if attempt < settings.retry_times:
time.sleep(settings.retry_sleep_ms / 1000.0 * max(1, attempt + 1))
continue
return {
"success": False,
"error": last_error,
"cost_ms": int((time.monotonic() - start) * 1000),
}
content = (
self._extract_responses_text(data)
if settings.openai_wire_api == "responses"
else str((((data.get("choices") or [{}])[0].get("message") or {}).get("content")) or "")
)
if response.status_code == 200 and content.strip():
usage = data.get("usage") or {}
tokens = usage.get("total_tokens") or (usage.get("input_tokens", 0) + usage.get("output_tokens", 0))
return {
"success": True,
"content": content.strip(),
"tokens": int(tokens or 0),
"cost_ms": int((time.monotonic() - start) * 1000),
}
last_error = str(((data.get("error") or {}).get("message")) or f"HTTP {response.status_code}")
if attempt < settings.retry_times and (response.status_code == 429 or 500 <= response.status_code <= 599):
time.sleep(settings.retry_sleep_ms / 1000.0 * max(1, attempt + 1))
continue
return {
"success": False,
"error": last_error,
"cost_ms": int((time.monotonic() - start) * 1000),
}
return {"success": False, "error": last_error, "cost_ms": int((time.monotonic() - start) * 1000)}
def _build_user_message(
self,
candidate: CommentCandidate,
virtual_user: VirtualUser,
persona: Dict[str, str],
) -> str:
sources = "、".join(candidate.selection_sources)
tags = "、".join(candidate.context_tags) if candidate.context_tags else "无"
title = candidate.title or "无标题"
body = candidate.body or "无正文"
return (
f"账号昵称:{virtual_user.nickname}\n"
f"人物角色:{persona['role']}\n"
f"语气风格:{persona['tone']}\n"
f"发言场景:{persona['scene']}\n"
f"关注重点:{persona['focus']}\n"
f"来源标签:{sources}\n"
f"社区标签:{tags}\n"
f"标题:{title}\n"
f"内容:{body}\n\n"
"请直接输出1条适合发布在评论区的中文评论。"
)
@staticmethod
def _build_endpoint(base_url: str, wire_api: str) -> str:
base = str(base_url or DEFAULT_OPENAI_BASE_URL).rstrip("/")
if wire_api == "responses":
if base.endswith("/responses"):
return base
if base.endswith("/v1"):
return f"{base}/responses"
return f"{base}/v1/responses"
if base.endswith("/chat/completions"):
return base
if base.endswith("/v1"):
return f"{base}/chat/completions"
return f"{base}/v1/chat/completions"
@staticmethod
def _build_payload(system_prompt: str, user_message: str, settings: AiCommentSettings) -> Dict[str, Any]:
if settings.openai_wire_api == "responses":
payload: Dict[str, Any] = {
"model": settings.openai_model,
"instructions": system_prompt,
"input": [
{
"role": "user",
"content": [{"type": "input_text", "text": user_message}],
}
],
"max_output_tokens": settings.max_tokens,
"temperature": settings.temperature,
}
if settings.openai_reasoning_effort:
payload["reasoning"] = {"effort": settings.openai_reasoning_effort}
if _truthy(settings.openai_disable_response_storage):
payload["store"] = False
return payload
return {
"model": settings.openai_model,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_message},
],
"max_tokens": settings.max_tokens,
"temperature": settings.temperature,
}
@staticmethod
def _extract_responses_text(data: Dict[str, Any]) -> str:
if data.get("output_text"):
return str(data["output_text"]).strip()
texts: List[str] = []
for output in data.get("output") or []:
for content in output.get("content") or []:
if "text" in content:
texts.append(str(content["text"]))
return "\n".join(texts).strip()
@staticmethod
def _persona_by_key(persona_key: str) -> Dict[str, str]:
for profile in PERSONA_PROFILES:
if profile["key"] == persona_key:
return profile
return PERSONA_PROFILES[0]
class AiCommentDispatcher:
def __init__(self, repo, client: Optional[AiCommentLLMClient] = None):
self.repo = repo
self.client = client or AiCommentLLMClient()
def run(self, settings: AiCommentSettings) -> Dict[str, Any]:
self.repo.ensure_schema()
users = self.repo.ensure_virtual_users(settings.user_pool_size)
if not settings.enabled:
return self._summary("skipped", "disabled", 0, 0, 0)
if not users:
return self._summary("skipped", "no_virtual_users", 0, 0, 0)
day_start = self._today_start()
day_end = day_start + 86400
current_success = self.repo.count_daily_success(day_start, day_end)
if current_success >= settings.daily_cap:
return self._summary("skipped", "daily_cap_reached", 0, 0, 0)
article_candidates = self.repo.load_article_candidates(settings)
post_candidates = self.repo.load_post_candidates(settings)
available_quota = max(0, settings.daily_cap - current_success)
selected = select_balanced_candidates(
article_candidates,
post_candidates,
min(settings.per_run, available_quota),
)
if not selected:
return self._summary("skipped", "no_candidates", 0, 0, 0)
start_offset = current_success % len(users)
task_items = []
for index, candidate in enumerate(selected):
virtual_user = users[(start_offset + index) % len(users)]
task = self.repo.create_task(candidate, virtual_user, virtual_user.persona_key)
task_items.append((task, candidate, virtual_user))
results = self._generate_comments(task_items, settings)
success_count = 0
failed_count = 0
for task, candidate, virtual_user, result in results:
content = ""
if result.get("success"):
content = self._clean_comment(str(result.get("content") or ""))
if not content:
content = self._build_fallback_comment(candidate)
if content:
self.repo.mark_task_success(task, content)
success_count += 1
else:
self.repo.mark_task_failed(task, str(result.get("error") or "生成评论失败"))
failed_count += 1
return self._summary("success", "", len(selected), success_count, failed_count)
def _generate_comments(self, task_items, settings: AiCommentSettings):
if settings.concurrency <= 1 or len(task_items) <= 1:
return [
(task, candidate, virtual_user, self.client.generate_comment(candidate, virtual_user, virtual_user.persona_key, settings))
for task, candidate, virtual_user in task_items
]
results = [None] * len(task_items)
with concurrent.futures.ThreadPoolExecutor(max_workers=settings.concurrency) as executor:
future_map = {
executor.submit(
self.client.generate_comment,
candidate,
virtual_user,
virtual_user.persona_key,
settings,
): (index, task, candidate, virtual_user)
for index, (task, candidate, virtual_user) in enumerate(task_items)
}
for future in concurrent.futures.as_completed(future_map):
index, task, candidate, virtual_user = future_map[future]
try:
result = future.result()
except Exception as exc: # noqa: BLE001
result = {"success": False, "error": str(exc)}
results[index] = (task, candidate, virtual_user, result)
return [item for item in results if item is not None]
@staticmethod
def _clean_comment(content: str) -> str:
value = re.sub(r"^评论[:]\s*", "", content.strip())
value = re.sub(r'^[“"\'‘’]+|[”"\'‘’]+$', "", value)
value = re.sub(r"\s+", " ", value).strip(" \t\r\n,。,.!?;:")
if len(value) < 8:
return ""
return value[:120]
@staticmethod
def _build_fallback_comment(candidate: CommentCandidate) -> str:
focus = candidate.title or candidate.body
focus = normalize_text(focus, limit=22)
if not focus:
return ""
if candidate.target_type == "article":
return f"{focus}这条信息量不小,后续变化值得继续盯着看。"
return f"{focus}这个角度说得挺到位,后面走势还得继续观察。"
@staticmethod
def _today_start() -> int:
now = time.localtime()
return int(time.mktime((now.tm_year, now.tm_mon, now.tm_mday, 0, 0, 0, now.tm_wday, now.tm_yday, now.tm_isdst)))
@staticmethod
def _summary(status: str, reason: str, selected_count: int, success_count: int, failed_count: int) -> Dict[str, Any]:
if status == "success":
summary_text = f"AI评论投放完成:选中{selected_count}条,成功{success_count}条,失败{failed_count}条"
else:
summary_text = f"AI评论投放跳过:{reason}"
return {
"success": True,
"status": status,
"reason": reason,
"selected_count": selected_count,
"success_count": success_count,
"failed_count": failed_count,
"candidate_count": selected_count,
"saved_count": success_count,
"count": success_count,
"summary_text": summary_text,
}
class AiCommentRepository:
def __init__(self, db_config: Dict[str, Any]):
self.prefix = str(db_config.get("prefix") or "la_")
self.conn = pymysql.connect(
host=db_config["host"],
port=int(db_config["port"]),
user=db_config["user"],
password=db_config["password"],
database=db_config["database"],
charset=db_config.get("charset", "utf8mb4"),
autocommit=False,
cursorclass=pymysql.cursors.DictCursor,
)
self._settings_cache: Optional[Dict[str, str]] = None
self._virtual_users_cache: Optional[List[VirtualUser]] = None
def close(self):
self.conn.close()
@property
def ai_task_table(self) -> str:
return f"`{self.prefix}ai_comment_task`"
def ensure_schema(self):
sql = f"""
CREATE TABLE IF NOT EXISTS {self.ai_task_table} (
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
`target_type` VARCHAR(20) NOT NULL DEFAULT '' COMMENT 'article/post',
`target_id` BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '内容ID',
`virtual_user_id` BIGINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '虚拟账号ID',
`persona_key` VARCHAR(50) NOT NULL DEFAULT '' COMMENT '人设标识',
`selection_sources` TEXT NULL COMMENT '来源集合',
`comment_content` TEXT NULL COMMENT '评论正文',
`status` TINYINT UNSIGNED NOT NULL DEFAULT 0 COMMENT '0待执行 1成功 2失败',
`error_msg` VARCHAR(255) NOT NULL DEFAULT '' COMMENT '错误原因',
`create_time` INT UNSIGNED NOT NULL DEFAULT 0,
`update_time` INT UNSIGNED NOT NULL DEFAULT 0,
PRIMARY KEY (`id`),
KEY `idx_target_status` (`target_type`, `target_id`, `status`),
KEY `idx_virtual_user` (`virtual_user_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Docker AI评论任务表'
"""
with self.conn.cursor() as cursor:
cursor.execute(sql)
self.conn.commit()
def load_ai_config_map(self) -> Dict[str, str]:
if self._settings_cache is not None:
return dict(self._settings_cache)
with self.conn.cursor() as cursor:
cursor.execute(f"SELECT name, value FROM `{self.prefix}ai_config`")
self._settings_cache = {str(row['name']): str(row['value'] or '') for row in cursor.fetchall()}
return dict(self._settings_cache)
def ensure_virtual_users(self, target_count: int) -> List[VirtualUser]:
if self._virtual_users_cache is not None and len(self._virtual_users_cache) >= target_count:
return list(self._virtual_users_cache[:target_count])
existing = self._load_virtual_users()
avatar = existing[0]["avatar"] if existing else self._load_site_config("default_image", "user_avatar")
salt = self._load_site_config("project", "unique_identification") or "sport-era"
indexed = {self._parse_account_index(row["account"]): row for row in existing if self._parse_account_index(row["account"]) > 0}
now = int(time.time())
with self.conn.cursor() as cursor:
for index in range(1, target_count + 1):
persona = self._persona_for_index(index)
account = f"{BOT_ACCOUNT_PREFIX}{index:04d}"
nickname = self._build_bot_nickname(index)
row = indexed.get(index)
if row:
if row["nickname"] != nickname:
cursor.execute(
f"UPDATE `{self.prefix}user` SET nickname=%s, update_time=%s WHERE id=%s",
(nickname, now, row["id"]),
)
continue
password = self._create_password(self._random_password(), salt)
sn = self._create_user_sn(cursor)
cursor.execute(
f"""
INSERT INTO `{self.prefix}user`
(`sn`, `avatar`, `real_name`, `nickname`, `account`, `password`, `mobile`, `sex`, `channel`,
`is_disable`, `is_new_user`, `user_money`, `total_recharge_amount`, `create_time`, `update_time`)
VALUES (%s, %s, '', %s, %s, %s, '', 0, 0, 0, 0, 0, 0, %s, %s)
""",
(sn, avatar, nickname, account, password, now, now),
)
self.conn.commit()
rows = self._load_virtual_users(limit=target_count)
users = []
for row in rows:
index = self._parse_account_index(row["account"])
persona = self._persona_for_index(index)
users.append(VirtualUser(int(row["id"]), row["account"], self._build_bot_nickname(index), persona["key"]))
self._virtual_users_cache = list(users)
return users[:target_count]
def count_daily_success(self, day_start: int, day_end: int) -> int:
with self.conn.cursor() as cursor:
cursor.execute(
f"SELECT COUNT(*) AS c FROM {self.ai_task_table} WHERE status=1 AND create_time >= %s AND create_time < %s",
(day_start, day_end),
)
row = cursor.fetchone() or {}
return int(row.get("c") or 0)
def load_settings(self) -> AiCommentSettings:
return AiCommentSettings.from_config_map(self.load_ai_config_map())
def load_article_candidates(self, settings: AiCommentSettings) -> List[CommentCandidate]:
bot_user_ids = self._load_bot_user_ids(settings.user_pool_size)
candidates: List[CommentCandidate] = []
with self.conn.cursor() as cursor:
cursor.execute(f"SELECT id, name FROM `{self.prefix}article_cate` WHERE is_show=1 ORDER BY sort DESC, id DESC")
categories = cursor.fetchall()
for category in categories:
cursor.execute(
f"""
SELECT id, title, `desc`, content, create_time, published_at, click_actual, click_virtual
FROM `{self.prefix}article`
WHERE is_show=1 AND delete_time IS NULL AND cid=%s
ORDER BY published_at DESC, id DESC
LIMIT 100
""",
(category["id"],),
)
for row in cursor.fetchall():
candidates.append(
CommentCandidate(
"article",
int(row["id"]),
normalize_text(str(row.get("title") or ""), limit=120),
normalize_text(f"{row.get('desc') or ''}\n{row.get('content') or ''}"),
(f"cate:{int(category['id'])}",),
created_at=int(row.get("create_time") or 0),
rank_value=int(row.get("click_actual") or 0) + int(row.get("click_virtual") or 0),
metadata={"category_names": (str(category["name"]),)},
)
)
cursor.execute(
f"""
SELECT id, title, `desc`, content, create_time, click_actual, click_virtual
FROM `{self.prefix}article`
WHERE is_show=1 AND delete_time IS NULL AND is_recommend=1
ORDER BY published_at DESC, id DESC
LIMIT 10
"""
)
for row in cursor.fetchall():
candidates.append(
CommentCandidate(
"article",
int(row["id"]),
normalize_text(str(row.get("title") or ""), limit=120),
normalize_text(f"{row.get('desc') or ''}\n{row.get('content') or ''}"),
("top",),
created_at=int(row.get("create_time") or 0),
rank_value=int(row.get("click_actual") or 0) + int(row.get("click_virtual") or 0),
)
)
day_start = AiCommentDispatcher._today_start()
cursor.execute(
f"""
SELECT id, title, `desc`, content, create_time, click_actual, click_virtual
FROM `{self.prefix}article`
WHERE is_show=1 AND delete_time IS NULL AND create_time >= %s
ORDER BY (click_actual + click_virtual) DESC, id DESC
LIMIT 20
""",
(day_start,),
)
for row in cursor.fetchall():
candidates.append(
CommentCandidate(
"article",
int(row["id"]),
normalize_text(str(row.get("title") or ""), limit=120),
normalize_text(f"{row.get('desc') or ''}\n{row.get('content') or ''}"),
("today_hot",),
created_at=int(row.get("create_time") or 0),
rank_value=int(row.get("click_actual") or 0) + int(row.get("click_virtual") or 0),
)
)
return self._filter_candidates(merge_candidates(candidates), bot_user_ids)
def load_post_candidates(self, settings: AiCommentSettings) -> List[CommentCandidate]:
bot_user_ids = self._load_bot_user_ids(settings.user_pool_size)
candidates: List[CommentCandidate] = []
with self.conn.cursor() as cursor:
cursor.execute(f"SELECT id, name FROM `{self.prefix}community_tag` WHERE status=1 ORDER BY post_count DESC, id DESC")
tags = cursor.fetchall()
for tag in tags:
cursor.execute(
f"""
SELECT p.id, p.content, p.ext, p.create_time, p.view_count
FROM `{self.prefix}community_post` p
INNER JOIN `{self.prefix}community_post_tag` pt ON pt.post_id = p.id
WHERE pt.tag_id=%s AND p.status=1 AND p.delete_time IS NULL
ORDER BY p.create_time DESC, p.id DESC
LIMIT 100
""",
(tag["id"],),
)
for row in cursor.fetchall():
ext = self._decode_json(row.get("ext"))
candidates.append(
CommentCandidate(
"post",
int(row["id"]),
"",
extract_post_text(str(row.get("content") or ""), ext),
(f"tag:{int(tag['id'])}",),
created_at=int(row.get("create_time") or 0),
rank_value=int(row.get("view_count") or 0),
context_tags=(str(tag["name"]),),
)
)
cursor.execute(
f"""
SELECT id, content, ext, create_time, view_count
FROM `{self.prefix}community_post`
WHERE status=1 AND delete_time IS NULL AND is_top=1
ORDER BY create_time DESC, id DESC
LIMIT 10
"""
)
for row in cursor.fetchall():
candidates.append(
CommentCandidate(
"post",
int(row["id"]),
"",
extract_post_text(str(row.get("content") or ""), self._decode_json(row.get("ext"))),
("top",),
created_at=int(row.get("create_time") or 0),
rank_value=int(row.get("view_count") or 0),
context_tags=tuple(self._load_post_tag_names(int(row["id"]))),
)
)
day_start = AiCommentDispatcher._today_start()
cursor.execute(
f"""
SELECT id, content, ext, create_time, view_count
FROM `{self.prefix}community_post`
WHERE status=1 AND delete_time IS NULL AND create_time >= %s
ORDER BY view_count DESC, id DESC
LIMIT 20
""",
(day_start,),
)
for row in cursor.fetchall():
candidates.append(
CommentCandidate(
"post",
int(row["id"]),
"",
extract_post_text(str(row.get("content") or ""), self._decode_json(row.get("ext"))),
("today_hot",),
created_at=int(row.get("create_time") or 0),
rank_value=int(row.get("view_count") or 0),
context_tags=tuple(self._load_post_tag_names(int(row["id"]))),
)
)
return self._filter_candidates(merge_candidates(candidates), bot_user_ids)
def create_task(self, candidate: CommentCandidate, virtual_user: VirtualUser, persona_key: str) -> DispatchTask:
now = int(time.time())
with self.conn.cursor() as cursor:
cursor.execute(
f"""
INSERT INTO {self.ai_task_table}
(`target_type`, `target_id`, `virtual_user_id`, `persona_key`, `selection_sources`,
`comment_content`, `status`, `error_msg`, `create_time`, `update_time`)
VALUES (%s, %s, %s, %s, %s, '', 0, '', %s, %s)
""",
(
candidate.target_type,
candidate.target_id,
virtual_user.user_id,
persona_key,
json.dumps(candidate.selection_sources, ensure_ascii=False),
now,
now,
),
)
task_id = int(cursor.lastrowid)
self.conn.commit()
return DispatchTask(task_id, candidate.target_type, candidate.target_id, virtual_user.user_id, persona_key, candidate.selection_sources)
def mark_task_success(self, task: DispatchTask, comment_content: str):
now = int(time.time())
with self.conn.cursor() as cursor:
try:
if task.target_type == "article":
cursor.execute(
f"""
INSERT INTO `{self.prefix}article_comment`
(`article_id`, `user_id`, `parent_id`, `reply_user_id`, `content`, `like_count`, `is_show`, `create_time`, `update_time`)
VALUES (%s, %s, 0, 0, %s, 0, 1, %s, %s)
""",
(task.target_id, task.virtual_user_id, comment_content, now, now),
)
cursor.execute(
f"UPDATE `{self.prefix}article` SET comment_count = comment_count + 1, update_time = %s WHERE id = %s",
(now, task.target_id),
)
else:
cursor.execute(
f"""
INSERT INTO `{self.prefix}community_comment`
(`post_id`, `user_id`, `parent_id`, `reply_user_id`, `content`, `like_count`, `status`, `create_time`)
VALUES (%s, %s, 0, 0, %s, 0, 1, %s)
""",
(task.target_id, task.virtual_user_id, comment_content, now),
)
cursor.execute(
f"UPDATE `{self.prefix}community_post` SET comment_count = comment_count + 1, update_time = %s WHERE id = %s",
(now, task.target_id),
)
cursor.execute(
f"UPDATE {self.ai_task_table} SET status=1, comment_content=%s, error_msg='', update_time=%s WHERE id=%s",
(comment_content, now, task.task_id),
)
self.conn.commit()
except Exception:
self.conn.rollback()
raise
def mark_task_failed(self, task: DispatchTask, message: str):
with self.conn.cursor() as cursor:
cursor.execute(
f"UPDATE {self.ai_task_table} SET status=2, error_msg=%s, update_time=%s WHERE id=%s",
(str(message or "")[:255], int(time.time()), task.task_id),
)
self.conn.commit()
def _load_virtual_users(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
sql = f"SELECT id, account, nickname, avatar FROM `{self.prefix}user` WHERE account LIKE %s AND delete_time IS NULL ORDER BY account ASC"
if limit:
sql += f" LIMIT {int(limit)}"
with self.conn.cursor() as cursor:
cursor.execute(sql, (f"{BOT_ACCOUNT_PREFIX}%",))
return list(cursor.fetchall())
def _load_bot_user_ids(self, target_count: int) -> List[int]:
users = self.ensure_virtual_users(target_count)
return [user.user_id for user in users]
def _load_site_config(self, config_type: str, name: str) -> str:
with self.conn.cursor() as cursor:
cursor.execute(
f"SELECT value FROM `{self.prefix}config` WHERE type=%s AND name=%s LIMIT 1",
(config_type, name),
)
row = cursor.fetchone() or {}
return str(row.get("value") or "")
def _create_user_sn(self, cursor) -> str:
while True:
sn = "9" + "".join(str(random.randint(1, 9)) for _ in range(8))
cursor.execute(f"SELECT id FROM `{self.prefix}user` WHERE sn=%s LIMIT 1", (sn,))
if not cursor.fetchone():
return sn
def _filter_candidates(self, candidates: Sequence[CommentCandidate], bot_user_ids: Sequence[int]) -> List[CommentCandidate]:
article_ids = [candidate.target_id for candidate in candidates if candidate.target_type == "article"]
post_ids = [candidate.target_id for candidate in candidates if candidate.target_type == "post"]
pending_keys = self._load_pending_task_keys(article_ids, post_ids)
commented_articles = self._load_commented_target_ids("article", article_ids, bot_user_ids)
commented_posts = self._load_commented_target_ids("post", post_ids, bot_user_ids)
filtered: List[CommentCandidate] = []
for candidate in candidates:
key = (candidate.target_type, candidate.target_id)
if key in pending_keys:
continue
if candidate.target_type == "article" and candidate.target_id in commented_articles:
continue
if candidate.target_type == "post" and candidate.target_id in commented_posts:
continue
if not candidate.title and not candidate.body:
continue
filtered.append(candidate)
return filtered
def _load_pending_task_keys(self, article_ids: Sequence[int], post_ids: Sequence[int]) -> set[Tuple[str, int]]:
keys: set[Tuple[str, int]] = set()
with self.conn.cursor() as cursor:
if article_ids:
placeholders = ",".join(["%s"] * len(article_ids))
cursor.execute(
f"SELECT target_type, target_id FROM {self.ai_task_table} WHERE target_type='article' AND target_id IN ({placeholders}) AND status IN (0, 1)",
tuple(article_ids),
)
keys.update((str(row["target_type"]), int(row["target_id"])) for row in cursor.fetchall())
if post_ids:
placeholders = ",".join(["%s"] * len(post_ids))
cursor.execute(
f"SELECT target_type, target_id FROM {self.ai_task_table} WHERE target_type='post' AND target_id IN ({placeholders}) AND status IN (0, 1)",
tuple(post_ids),
)
keys.update((str(row["target_type"]), int(row["target_id"])) for row in cursor.fetchall())
return keys
def _load_commented_target_ids(self, target_type: str, target_ids: Sequence[int], bot_user_ids: Sequence[int]) -> set[int]:
if not target_ids or not bot_user_ids:
return set()
placeholders_targets = ",".join(["%s"] * len(target_ids))
placeholders_users = ",".join(["%s"] * len(bot_user_ids))
if target_type == "article":
sql = (
f"SELECT DISTINCT article_id AS target_id FROM `{self.prefix}article_comment` "
f"WHERE article_id IN ({placeholders_targets}) AND user_id IN ({placeholders_users}) AND is_show=1 AND delete_time IS NULL"
)
else:
sql = (
f"SELECT DISTINCT post_id AS target_id FROM `{self.prefix}community_comment` "
f"WHERE post_id IN ({placeholders_targets}) AND user_id IN ({placeholders_users}) AND status=1 AND delete_time IS NULL"
)
params = tuple(target_ids) + tuple(bot_user_ids)
with self.conn.cursor() as cursor:
cursor.execute(sql, params)
return {int(row["target_id"]) for row in cursor.fetchall()}
def _load_post_tag_names(self, post_id: int) -> List[str]:
with self.conn.cursor() as cursor:
cursor.execute(
f"""
SELECT t.name
FROM `{self.prefix}community_tag` t
INNER JOIN `{self.prefix}community_post_tag` pt ON pt.tag_id = t.id
WHERE pt.post_id=%s
ORDER BY t.id ASC
""",
(post_id,),
)
return [str(row["name"]) for row in cursor.fetchall()]
@staticmethod
def _decode_json(value: Any) -> Dict[str, Any]:
if isinstance(value, dict):
return value
if not value:
return {}
try:
decoded = json.loads(value)
except Exception: # noqa: BLE001
return {}
return decoded if isinstance(decoded, dict) else {}
@staticmethod
def _parse_account_index(account: str) -> int:
match = re.search(r"(\d+)$", account or "")
if not match:
return 0
return int(match.group(1))
@staticmethod
def _create_password(plaintext: str, salt: str) -> str:
inner = hashlib.md5(f"{plaintext}{salt}".encode("utf-8")).hexdigest()
return hashlib.md5(f"{salt}{inner}".encode("utf-8")).hexdigest()
@staticmethod
def _random_password() -> str:
return hashlib.md5(str(random.random()).encode("utf-8")).hexdigest()[:12]
@staticmethod
def _build_bot_nickname(index: int) -> str:
zero_based = max(0, index - 1)
prefix = BOT_HANDLE_PREFIXES[zero_based % len(BOT_HANDLE_PREFIXES)]
suffix = BOT_HANDLE_SUFFIXES[(zero_based // len(BOT_HANDLE_PREFIXES)) % len(BOT_HANDLE_SUFFIXES)]
return f"{prefix}{suffix}"
@staticmethod
def _persona_for_index(index: int) -> Dict[str, str]:
zero_based = max(0, index - 1)
return PERSONA_PROFILES[zero_based % len(PERSONA_PROFILES)]
def run(db_config: Dict[str, Any]) -> Dict[str, Any]:
repo = AiCommentRepository(db_config)
try:
dispatcher = AiCommentDispatcher(repo, AiCommentLLMClient())
settings = repo.load_settings()
return dispatcher.run(settings)
finally:
repo.close()