fix: prevent future article timestamps from pinning feeds

2026-08-03 23:46:38 +08:00
parent fabd378415
commit 450efec14d
6 changed files with 134 additions and 3 deletions
@@ -0,0 +1,29 @@
+"""文章发布时间规范化。"""
+
+from datetime import datetime, timedelta
+from typing import Any, Optional
+
+
+MAX_FUTURE_OFFSET = timedelta(days=2)
+
+
+def normalize_published_at(value: Any, now: Optional[datetime] = None) -> str:
+    """返回可安全用于资讯排序的发布时间，明显异常时返回空字符串。"""
+    raw = str(value or "").strip()
+    if not raw:
+        return ""
+
+    normalized = raw.replace("T", " ").removesuffix("Z")
+    try:
+        parsed = datetime.fromisoformat(normalized)
+    except ValueError:
+        return ""
+
+    if parsed.tzinfo is not None:
+        parsed = parsed.replace(tzinfo=None)
+
+    reference = now or datetime.now()
+    if parsed > reference + MAX_FUTURE_OFFSET:
+        return ""
+
+    return parsed.strftime("%Y-%m-%d %H:%M:%S")
@@ -14,6 +14,7 @@ import redis.asyncio as aioredis
 from loguru import logger

 from src.core.config import get_config
+from src.parser.publish_time import normalize_published_at


 class Database:
@@ -700,7 +701,7 @@ class Database:
                        author = str(author_info.get("name", ""))[:255]
                    else:
                        author = str(a.get("author_name", "") or "")[:255]
-                    published_at = str(a.get("published_at", ""))[:30]
+                    published_at = normalize_published_at(a.get("published_at", ""))[:30]
                    category = str(a.get("category", "") or "")[:50]
                    source_url = str(a.get("share", "") or "")[:500]
                    comment_count = int(a.get("comments_total", 0) or 0)
@@ -720,6 +721,7 @@ class Database:
                        thumb=IF(VALUES(thumb)='', thumb, VALUES(thumb)),
                        duration=IF(VALUES(duration)='', duration, VALUES(duration)),
                        video_url=IF(VALUES(video_url)='', video_url, VALUES(video_url)),
+                        published_at=IF(VALUES(published_at)='', published_at, VALUES(published_at)),
                        category=IF(VALUES(category)='', category, VALUES(category)),
                        sort=IF(VALUES(sort)=0, sort, VALUES(sort)),
                        comment_count=VALUES(comment_count),
@@ -0,0 +1,48 @@
+import importlib.util
+import pathlib
+import unittest
+from datetime import datetime
+
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+MODULE_PATH = ROOT / "src" / "parser" / "publish_time.py"
+
+
+def load_module():
+    if not MODULE_PATH.exists():
+        raise AssertionError("缺少文章发布时间规范化模块")
+    spec = importlib.util.spec_from_file_location("publish_time", MODULE_PATH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class ArticlePublishTimeTest(unittest.TestCase):
+    def setUp(self):
+        self.module = load_module()
+        self.now = datetime(2026, 8, 3, 15, 0, 0)
+
+    def test_keeps_normal_publish_time_within_timezone_tolerance(self):
+        value = self.module.normalize_published_at(
+            "2026-08-03 23:01:04",
+            now=self.now,
+        )
+
+        self.assertEqual(value, "2026-08-03 23:01:04")
+
+    def test_rejects_clearly_future_publish_time(self):
+        value = self.module.normalize_published_at(
+            "2031-07-21 04:50:09",
+            now=self.now,
+        )
+
+        self.assertEqual(value, "")
+
+    def test_rejects_unparseable_publish_time(self):
+        value = self.module.normalize_published_at("not-a-date", now=self.now)
+
+        self.assertEqual(value, "")
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,8 @@
+-- 修复明显晚于文章入库时间的异常发布时间。
+-- 条件以创建时间后 2 天为容差，兼容数据源与数据库时区差异；重复执行不会再次命中。
+UPDATE `la_article`
+SET `published_at` = DATE_FORMAT(FROM_UNIXTIME(create_time), '%Y-%m-%d %H:%i:%s')
+WHERE `create_time` IS NOT NULL
+  AND `published_at` REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$'
+  AND STR_TO_DATE(`published_at`, '%Y-%m-%d %H:%i:%s')
+      > DATE_ADD(FROM_UNIXTIME(create_time), INTERVAL 2 DAY);
@@ -0,0 +1,30 @@
+<?php
+
+$root = dirname(__DIR__, 2);
+$listsFile = $root . '/server/app/api/lists/article/ArticleLists.php';
+$databaseFile = $root . '/docker/crawler/src/storage/database.py';
+$repairFile = $root . '/docs/sql/repair_article_future_published_at.sql';
+
+$listsSource = file_get_contents($listsFile);
+$databaseSource = file_get_contents($databaseFile);
+$repairSource = is_file($repairFile) ? file_get_contents($repairFile) : '';
+
+$checks = [
+    '文章列表定义安全发布时间排序方法' => str_contains($listsSource, 'buildSafePublishedOrder'),
+    '排序会解析标准发布时间' => str_contains($listsSource, "STR_TO_DATE(published_at, '%Y-%m-%d %H:%i:%s')"),
+    '排序拒绝超过两天的未来时间' => str_contains($listsSource, 'DATE_ADD(NOW(), INTERVAL 2 DAY)'),
+    '异常发布时间回退到创建时间' => str_contains($listsSource, 'FROM_UNIXTIME(create_time)'),
+    '采集入库前规范化发布时间' => str_contains($databaseSource, 'normalize_published_at'),
+    '重复采集允许刷新有效发布时间' => str_contains($databaseSource, 'published_at=IF(VALUES(published_at)'),
+    '提供幂等的历史未来时间修复 SQL' => str_contains($repairSource, 'DATE_ADD(FROM_UNIXTIME(create_time), INTERVAL 2 DAY)'),
+    '历史时间修复回退到创建时间' => str_contains($repairSource, 'DATE_FORMAT(FROM_UNIXTIME(create_time)'),
+];
+
+foreach ($checks as $label => $passed) {
+    if (!$passed) {
+        fwrite(STDERR, "FAIL: {$label}\n");
+        exit(1);
+    }
+}
+
+echo "文章发布时间排序与采集保护静态检查通过\n";
@@ -97,12 +97,12 @@ class ArticleLists extends BaseApiDataLists implements ListsSearchInterface
     */
    public function lists(): array
    {
-        $orderRaw = 'published_at desc, id desc';
+        $orderRaw = $this->buildSafePublishedOrder();
        $sortType = $this->params['sort'] ?? 'default';
        $cid = (int) ($this->params['cid'] ?? 0);
        // 最新排序
        if ($sortType == 'new') {
-            $orderRaw = 'published_at desc, id desc';
+            $orderRaw = $this->buildSafePublishedOrder();
        }
        // 最热排序
        if ($sortType == 'hot') {
@@ -140,6 +140,20 @@ class ArticleLists extends BaseApiDataLists implements ListsSearchInterface
    }


+    /**
+     * 明显超前的脏发布时间不参与置顶排序，回退到文章入库时间。
+     */
+    private function buildSafePublishedOrder(): string
+    {
+        return "CASE
+            WHEN published_at REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$'
+                AND STR_TO_DATE(published_at, '%Y-%m-%d %H:%i:%s') <= DATE_ADD(NOW(), INTERVAL 2 DAY)
+            THEN STR_TO_DATE(published_at, '%Y-%m-%d %H:%i:%s')
+            ELSE FROM_UNIXTIME(create_time)
+        END DESC, id DESC";
+    }
+
+
    /**
     * @notes 获取文章数量
     * @return int