deploy: auto commit server changes 2026-06-11 20:44:36

2026-06-11 20:44:36 +08:00
parent 9fb80f951a
commit e17a9832af
11 changed files with 1195 additions and 23 deletions
@@ -7,7 +7,7 @@ use app\common\validate\BaseValidate;
 class KbSyncJobValidate extends BaseValidate
 {
    protected $rule = [
-        'domain' => 'require|in:article,post,lottery,all',
+        'domain' => 'require|in:article,post,lottery,match,all',
    ];

    protected $message = [
@@ -22,8 +22,9 @@ class KbConsume extends Command
    protected function execute(Input $input, Output $output)
    {
        $batch = max(1, (int) $input->getOption('batch'));
-        KbSyncService::markLotteryBackfillJobs($batch);
        KbSyncService::markMatchBackfillJobs($batch);
+        KbSyncService::markArticleBackfillJobs($batch);
+        KbSyncService::markLotteryBackfillJobs($batch);

        $jobs = AiKbSyncJob::where('status', AiKbSyncJob::STATUS_PENDING)
            ->where('scheduled_at', '<=', time())
@@ -0,0 +1,49 @@
+<?php
+
+namespace app\common\command;
+
+use app\common\service\ai\KbRedisQueueService;
+use think\console\Command;
+use think\console\Input;
+use think\console\Output;
+use think\console\input\Option;
+
+class KbEnqueueRecent extends Command
+{
+    protected function configure()
+    {
+        $this->setName('kb:enqueue-recent')
+            ->setDescription('扫描最近更新内容并写入AI知识库Redis队列')
+            ->addOption('limit', null, Option::VALUE_OPTIONAL, '每个来源最多扫描数量', 1000)
+            ->addOption('minutes', null, Option::VALUE_OPTIONAL, '扫描最近多少分钟更新的数据', 30);
+    }
+
+    protected function execute(Input $input, Output $output)
+    {
+        $limit = max(1, (int) $input->getOption('limit'));
+        $minutes = max(1, (int) $input->getOption('minutes'));
+        $summary = KbRedisQueueService::enqueueRecent($limit, $minutes);
+
+        foreach ($summary['targets'] as $label => $target) {
+            $output->writeln(sprintf(
+                '[kb:enqueue-recent] %s scanned=%d queued=%d deduped=%d',
+                $label,
+                (int) $target['scanned'],
+                (int) $target['queued'],
+                (int) $target['deduped']
+            ));
+        }
+
+        $output->writeln(sprintf(
+            '[kb:enqueue-recent] stream=%s scanned=%d queued=%d deduped=%d limit=%d minutes=%d',
+            $summary['stream'],
+            (int) $summary['scanned'],
+            (int) $summary['queued'],
+            (int) $summary['deduped'],
+            (int) $summary['limit'],
+            (int) $summary['minutes']
+        ));
+
+        return 0;
+    }
+}
@@ -43,7 +43,15 @@ class KbRebuild extends Command
            }
        }

-        KbSyncService::markLotteryBackfillJobs();
+        if (in_array('article', $targets, true)) {
+            KbSyncService::markArticleBackfillJobs();
+        }
+        if (in_array('match', $targets, true)) {
+            KbSyncService::markMatchBackfillJobs();
+        }
+        if (in_array('lottery', $targets, true)) {
+            KbSyncService::markLotteryBackfillJobs();
+        }
        $output->writeln(sprintf('[kb:rebuild] success=%d failed=%d', $summary['success'], $summary['failed']));
        return 0;
    }
@@ -0,0 +1,235 @@
+<?php
+
+namespace app\common\service\ai;
+
+use think\facade\Db;
+
+class KbRedisQueueService
+{
+    public const STREAM_KEY = 'ai:kb:sync:stream';
+    public const DEDUPE_PREFIX = 'ai:kb:sync:dedupe:';
+
+    private const DEFAULT_DEDUPE_TTL = 1800;
+
+    public static function enqueueRecent(int $limit = 1000, int $minutes = 30): array
+    {
+        $limit = max(1, $limit);
+        $minutes = max(1, $minutes);
+        $cutoff = time() - ($minutes * 60);
+        $redis = self::redis();
+        $summary = [
+            'stream' => self::STREAM_KEY,
+            'limit' => $limit,
+            'minutes' => $minutes,
+            'scanned' => 0,
+            'queued' => 0,
+            'deduped' => 0,
+            'targets' => [],
+        ];
+
+        foreach (self::targets() as $target) {
+            $result = self::enqueueTarget($redis, $target, $limit, $cutoff, $minutes * 60);
+            $summary['targets'][$target['label']] = $result;
+            $summary['scanned'] += $result['scanned'];
+            $summary['queued'] += $result['queued'];
+            $summary['deduped'] += $result['deduped'];
+        }
+
+        return $summary;
+    }
+
+    private static function enqueueTarget(\Redis $redis, array $target, int $limit, int $cutoff, int $dedupeTtl): array
+    {
+        $result = [
+            'domain' => $target['domain'],
+            'subtype' => $target['subtype'],
+            'table' => $target['table'],
+            'scanned' => 0,
+            'queued' => 0,
+            'deduped' => 0,
+        ];
+
+        $rows = self::recentUnsyncedRows($target, $limit, $cutoff);
+        $result['scanned'] = count($rows);
+
+        foreach ($rows as $row) {
+            $sourceId = (int) ($row['source_id'] ?? 0);
+            if ($sourceId <= 0) {
+                continue;
+            }
+
+            $dedupeKey = self::dedupeKey($target['domain'], $target['subtype'], $sourceId);
+            $setResult = $redis->rawCommand('SET', $dedupeKey, '1', 'EX', (string) max(60, $dedupeTtl), 'NX');
+            if ($setResult !== true && strtoupper((string) $setResult) !== 'OK') {
+                $result['deduped']++;
+                continue;
+            }
+
+            $messageId = self::xAdd($redis, [
+                'domain' => $target['domain'],
+                'subtype' => $target['subtype'],
+                'source_id' => (string) $sourceId,
+                'source_updated_at' => (string) (int) ($row['source_updated_at'] ?? 0),
+                'priority' => (string) (int) $target['priority'],
+                'trace_id' => self::traceId($target['label'], $sourceId),
+            ]);
+
+            if ($messageId !== false && $messageId !== null && $messageId !== '') {
+                $result['queued']++;
+            }
+        }
+
+        return $result;
+    }
+
+    private static function recentUnsyncedRows(array $target, int $limit, int $cutoff): array
+    {
+        $sourceUpdatedExpr = $target['source_updated_expr'];
+        $query = Db::name($target['table'])->alias('s')
+            ->leftJoin(
+                'ai_kb_document d',
+                "d.domain = '{$target['domain']}' AND d.subtype = '{$target['subtype']}' AND d.source_id = s.id"
+            )
+            ->where('s.id', '>', 0)
+            ->whereRaw("{$sourceUpdatedExpr} >= {$cutoff}")
+            ->whereRaw("(d.id IS NULL OR d.is_active <> 1 OR d.source_updated_at < {$sourceUpdatedExpr})");
+
+        self::applySourceFilters($query, $target);
+
+        return $query
+            ->fieldRaw("s.id AS source_id, {$sourceUpdatedExpr} AS source_updated_at")
+            ->orderRaw("{$sourceUpdatedExpr} DESC, s.id DESC")
+            ->limit($limit)
+            ->select()
+            ->toArray();
+    }
+
+    private static function applySourceFilters($query, array $target): void
+    {
+        if (($target['domain'] ?? '') === 'article') {
+            $query->where('s.is_show', 1)->whereNull('s.delete_time');
+        }
+
+        if (($target['domain'] ?? '') === 'match') {
+            $query->where('s.is_show', 1)->whereNull('s.delete_time');
+        }
+
+        if (($target['domain'] ?? '') === 'post') {
+            $query->where('s.status', 1)->whereNull('s.delete_time');
+        }
+
+        foreach (($target['where'] ?? []) as $field => $value) {
+            $query->where('s.' . $field, $value);
+        }
+    }
+
+    private static function targets(): array
+    {
+        return [
+            [
+                'label' => 'crypto_article',
+                'domain' => 'article',
+                'subtype' => 'article',
+                'table' => 'article',
+                'priority' => 82,
+                'source_updated_expr' => 'COALESCE(s.update_time, s.create_time, 0)',
+                'where' => ['cid' => 9],
+            ],
+            [
+                'label' => 'article',
+                'domain' => 'article',
+                'subtype' => 'article',
+                'table' => 'article',
+                'priority' => 85,
+                'source_updated_expr' => 'COALESCE(s.update_time, s.create_time, 0)',
+            ],
+            [
+                'label' => 'lottery_draw_result',
+                'domain' => 'lottery',
+                'subtype' => 'draw_result',
+                'table' => 'lottery_draw_result',
+                'priority' => 80,
+                'source_updated_expr' => 'COALESCE(s.update_time, s.create_time, 0)',
+            ],
+            [
+                'label' => 'lottery_ai_history',
+                'domain' => 'lottery',
+                'subtype' => 'ai_history',
+                'table' => 'lottery_ai_analysis',
+                'priority' => 90,
+                'source_updated_expr' => 'COALESCE(s.update_time, s.create_time, 0)',
+            ],
+            [
+                'label' => 'match',
+                'domain' => 'match',
+                'subtype' => 'event',
+                'table' => 'match',
+                'priority' => 60,
+                'source_updated_expr' => 'COALESCE(s.update_time, s.create_time, s.match_time, 0)',
+            ],
+            [
+                'label' => 'post',
+                'domain' => 'post',
+                'subtype' => 'post',
+                'table' => 'community_post',
+                'priority' => 75,
+                'source_updated_expr' => 'COALESCE(s.update_time, s.create_time, 0)',
+            ],
+        ];
+    }
+
+    private static function redis(): \Redis
+    {
+        if (!class_exists(\Redis::class)) {
+            throw new \RuntimeException('Redis 扩展不可用，无法写入 AI 知识库 Redis 队列');
+        }
+
+        $options = (array) config('cache.stores.redis');
+        $redis = new \Redis();
+        $host = (string) (env('AI_KB_REDIS_HOST', '') ?: ($options['host'] ?? '127.0.0.1'));
+        $port = (int) (env('AI_KB_REDIS_PORT', '') ?: ($options['port'] ?? 6379));
+        $timeout = (float) ($options['timeout'] ?? 0);
+        $redis->connect($host, $port, $timeout);
+
+        $password = (string) (env('AI_KB_REDIS_PASSWORD', '') ?: ($options['password'] ?? ''));
+        if ($password !== '') {
+            $redis->auth($password);
+        }
+
+        $select = (int) (env('AI_KB_REDIS_DB', '') ?: 4);
+        $redis->select($select);
+
+        return $redis;
+    }
+
+    private static function xAdd(\Redis $redis, array $fields)
+    {
+        if (method_exists($redis, 'xAdd')) {
+            return $redis->xAdd(self::STREAM_KEY, '*', $fields);
+        }
+
+        $args = ['XADD', self::STREAM_KEY, '*'];
+        foreach ($fields as $field => $value) {
+            $args[] = (string) $field;
+            $args[] = (string) $value;
+        }
+
+        return $redis->rawCommand(...$args);
+    }
+
+    private static function dedupeKey(string $domain, string $subtype, int $sourceId): string
+    {
+        return self::DEDUPE_PREFIX . $domain . ':' . $subtype . ':' . $sourceId;
+    }
+
+    private static function traceId(string $label, int $sourceId): string
+    {
+        try {
+            $suffix = bin2hex(random_bytes(4));
+        } catch (\Throwable $e) {
+            $suffix = (string) mt_rand(100000, 999999);
+        }
+
+        return $label . '-' . $sourceId . '-' . date('YmdHis') . '-' . $suffix;
+    }
+}
@@ -395,8 +395,8 @@ class KbService
                'published_at' => (string) ($data['published_at'] ?? ''),
                'is_video' => (int) ($data['is_video'] ?? 0),
            ],
-            'source_created_at' => (int) ($data['create_time'] ?? 0),
-            'source_updated_at' => (int) ($data['update_time'] ?? $data['create_time'] ?? 0),
+            'source_created_at' => self::sourceTimestamp($data['create_time'] ?? 0),
+            'source_updated_at' => self::sourceTimestamp($data['update_time'] ?? 0, $data['create_time'] ?? 0),
        ];
    }

@@ -435,8 +435,8 @@ class KbService
                'user_id' => (int) ($data['user_id'] ?? 0),
                'tags' => array_values($tags),
            ],
-            'source_created_at' => (int) ($data['create_time'] ?? 0),
-            'source_updated_at' => (int) ($data['update_time'] ?? $data['create_time'] ?? 0),
+            'source_created_at' => self::sourceTimestamp($data['create_time'] ?? 0),
+            'source_updated_at' => self::sourceTimestamp($data['update_time'] ?? 0, $data['create_time'] ?? 0),
        ];
    }

@@ -482,8 +482,8 @@ class KbService
                'game_name' => (string) ($game['name'] ?? ''),
                'draw_time' => (string) ($data['draw_time'] ?? ''),
            ],
-            'source_created_at' => (int) ($data['create_time'] ?? 0),
-            'source_updated_at' => (int) ($data['update_time'] ?? $data['create_time'] ?? 0),
+            'source_created_at' => self::sourceTimestamp($data['create_time'] ?? 0),
+            'source_updated_at' => self::sourceTimestamp($data['update_time'] ?? 0, $data['create_time'] ?? 0),
        ];
    }

@@ -543,8 +543,8 @@ class KbService
                'match_time' => $matchTime,
                'status' => (int) ($data['status'] ?? 0),
            ],
-            'source_created_at' => (int) ($data['create_time'] ?? $data['match_time'] ?? 0),
-            'source_updated_at' => (int) ($data['update_time'] ?? $data['match_time'] ?? $data['create_time'] ?? 0),
+            'source_created_at' => self::sourceTimestamp($data['create_time'] ?? 0, $data['match_time'] ?? 0),
+            'source_updated_at' => self::sourceTimestamp($data['update_time'] ?? 0, $data['match_time'] ?? ($data['create_time'] ?? 0)),
        ];
    }

@@ -576,11 +576,32 @@ class KbService
                'issue' => (string) ($data['issue'] ?? ''),
                'module' => (string) ($data['module'] ?? ''),
            ],
-            'source_created_at' => (int) ($data['create_time'] ?? 0),
-            'source_updated_at' => (int) ($data['update_time'] ?? $data['create_time'] ?? 0),
+            'source_created_at' => self::sourceTimestamp($data['create_time'] ?? 0),
+            'source_updated_at' => self::sourceTimestamp($data['update_time'] ?? 0, $data['create_time'] ?? 0),
        ];
    }

+    private static function sourceTimestamp($value, $fallback = 0): int
+    {
+        if (is_int($value)) {
+            return $value;
+        }
+
+        if (is_numeric($value)) {
+            return (int) $value;
+        }
+
+        $value = trim((string) $value);
+        if ($value !== '') {
+            $timestamp = strtotime($value);
+            if ($timestamp !== false) {
+                return $timestamp;
+            }
+        }
+
+        return is_array($fallback) ? 0 : self::sourceTimestamp($fallback);
+    }
+
    private static function buildKeywordText(array $payload): string
    {
        return implode(' ', array_filter([
@@ -55,41 +55,89 @@ class KbSyncService
        ]);
    }

+    public static function markArticleBackfillJobs(int $limit = 200): void
+    {
+        self::enqueueUnsyncedRows(
+            'article',
+            'article',
+            'article',
+            $limit,
+            85,
+            'COALESCE(s.update_time, s.create_time, 0)'
+        );
+    }
+
    public static function markLotteryBackfillJobs(int $limit = 200): void
    {
-        self::enqueueMissingLotteryRows('draw_result', 'lottery_draw_result', $limit);
-        self::enqueueMissingLotteryRows('ai_history', 'lottery_ai_analysis', $limit);
+        self::enqueueMissingLotteryRows('draw_result', 'lottery_draw_result', $limit, 80);
+        self::enqueueMissingLotteryRows('ai_history', 'lottery_ai_analysis', $limit, 90);
    }

    public static function markMatchBackfillJobs(int $limit = 200): void
    {
-        self::enqueueMissingRows('match', 'event', 'match', $limit);
+        self::enqueueUnsyncedRows(
+            'match',
+            'event',
+            'match',
+            $limit,
+            60,
+            'COALESCE(s.update_time, s.create_time, s.match_time, 0)'
+        );
    }

-    private static function enqueueMissingLotteryRows(string $subtype, string $table, int $limit): void
+    private static function enqueueMissingLotteryRows(string $subtype, string $table, int $limit, int $priority): void
    {
-        self::enqueueMissingRows('lottery', $subtype, $table, $limit);
+        self::enqueueUnsyncedRows(
+            'lottery',
+            $subtype,
+            $table,
+            $limit,
+            $priority,
+            'COALESCE(s.update_time, s.create_time, 0)'
+        );
    }

-    private static function enqueueMissingRows(string $domain, string $subtype, string $table, int $limit): void
-    {
+    private static function enqueueUnsyncedRows(
+        string $domain,
+        string $subtype,
+        string $table,
+        int $limit,
+        int $priority,
+        string $sourceUpdatedExpr
+    ): void {
        try {
-            $rows = Db::name($table)->alias('s')
+            $query = Db::name($table)->alias('s')
                ->leftJoin('ai_kb_document d', "d.domain = '{$domain}' AND d.subtype = '{$subtype}' AND d.source_id = s.id")
-                ->whereNull('d.id')
                ->where('s.id', '>', 0)
+                ->whereRaw("(d.id IS NULL OR d.is_active <> 1 OR d.source_updated_at < {$sourceUpdatedExpr})");
+
+            self::applySourceFilters($query, $domain);
+
+            $rows = $query
                ->order('s.id', 'desc')
                ->limit($limit)
                ->column('s.id');

            foreach ($rows as $sourceId) {
-                self::enqueue($domain, $subtype, (int) $sourceId, AiKbSyncJob::ACTION_UPSERT, 80);
+                self::enqueue($domain, $subtype, (int) $sourceId, AiKbSyncJob::ACTION_UPSERT, $priority);
            }
        } catch (\Throwable $e) {
            // Ignore in environments where KB tables are not ready yet.
        }
    }

+    private static function applySourceFilters($query, string $domain): void
+    {
+        if ($domain === 'article') {
+            $query->where('s.is_show', 1)->whereNull('s.delete_time');
+            return;
+        }
+
+        if ($domain === 'match') {
+            $query->where('s.is_show', 1);
+        }
+    }
+
    public static function markDocumentInactive(string $domain, string $subtype, int $sourceId): void
    {
        AiKbDocument::where([