From 450efec14d0d721d294394caee51946e29784580 Mon Sep 17 00:00:00 2001 From: hajimi Date: Mon, 3 Aug 2026 23:46:38 +0800 Subject: [PATCH] fix: prevent future article timestamps from pinning feeds --- docker/crawler/src/parser/publish_time.py | 29 +++++++++++ docker/crawler/src/storage/database.py | 4 +- .../tests/test_article_publish_time.py | 48 +++++++++++++++++++ .../repair_article_future_published_at.sql | 8 ++++ .../test_article_publish_sort_static.php | 30 ++++++++++++ server/app/api/lists/article/ArticleLists.php | 18 ++++++- 6 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 docker/crawler/src/parser/publish_time.py create mode 100644 docker/crawler/tests/test_article_publish_time.py create mode 100644 docs/sql/repair_article_future_published_at.sql create mode 100644 qa/backend/test_article_publish_sort_static.php diff --git a/docker/crawler/src/parser/publish_time.py b/docker/crawler/src/parser/publish_time.py new file mode 100644 index 00000000..8c9ce199 --- /dev/null +++ b/docker/crawler/src/parser/publish_time.py @@ -0,0 +1,29 @@ +"""文章发布时间规范化。""" + +from datetime import datetime, timedelta +from typing import Any, Optional + + +MAX_FUTURE_OFFSET = timedelta(days=2) + + +def normalize_published_at(value: Any, now: Optional[datetime] = None) -> str: + """返回可安全用于资讯排序的发布时间，明显异常时返回空字符串。""" + raw = str(value or "").strip() + if not raw: + return "" + + normalized = raw.replace("T", " ").removesuffix("Z") + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return "" + + if parsed.tzinfo is not None: + parsed = parsed.replace(tzinfo=None) + + reference = now or datetime.now() + if parsed > reference + MAX_FUTURE_OFFSET: + return "" + + return parsed.strftime("%Y-%m-%d %H:%M:%S") diff --git a/docker/crawler/src/storage/database.py b/docker/crawler/src/storage/database.py index 828b8b35..9639b35d 100644 --- a/docker/crawler/src/storage/database.py +++ b/docker/crawler/src/storage/database.py @@ -14,6 +14,7 @@ import redis.asyncio as aioredis from loguru import logger from src.core.config import get_config +from src.parser.publish_time import normalize_published_at class Database: @@ -700,7 +701,7 @@ class Database: author = str(author_info.get("name", ""))[:255] else: author = str(a.get("author_name", "") or "")[:255] - published_at = str(a.get("published_at", ""))[:30] + published_at = normalize_published_at(a.get("published_at", ""))[:30] category = str(a.get("category", "") or "")[:50] source_url = str(a.get("share", "") or "")[:500] comment_count = int(a.get("comments_total", 0) or 0) @@ -720,6 +721,7 @@ class Database: thumb=IF(VALUES(thumb)='', thumb, VALUES(thumb)), duration=IF(VALUES(duration)='', duration, VALUES(duration)), video_url=IF(VALUES(video_url)='', video_url, VALUES(video_url)), + published_at=IF(VALUES(published_at)='', published_at, VALUES(published_at)), category=IF(VALUES(category)='', category, VALUES(category)), sort=IF(VALUES(sort)=0, sort, VALUES(sort)), comment_count=VALUES(comment_count), diff --git a/docker/crawler/tests/test_article_publish_time.py b/docker/crawler/tests/test_article_publish_time.py new file mode 100644 index 00000000..b37d170f --- /dev/null +++ b/docker/crawler/tests/test_article_publish_time.py @@ -0,0 +1,48 @@ +import importlib.util +import pathlib +import unittest +from datetime import datetime + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "src" / "parser" / "publish_time.py" + + +def load_module(): + if not MODULE_PATH.exists(): + raise AssertionError("缺少文章发布时间规范化模块") + spec = importlib.util.spec_from_file_location("publish_time", MODULE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class ArticlePublishTimeTest(unittest.TestCase): + def setUp(self): + self.module = load_module() + self.now = datetime(2026, 8, 3, 15, 0, 0) + + def test_keeps_normal_publish_time_within_timezone_tolerance(self): + value = self.module.normalize_published_at( + "2026-08-03 23:01:04", + now=self.now, + ) + + self.assertEqual(value, "2026-08-03 23:01:04") + + def test_rejects_clearly_future_publish_time(self): + value = self.module.normalize_published_at( + "2031-07-21 04:50:09", + now=self.now, + ) + + self.assertEqual(value, "") + + def test_rejects_unparseable_publish_time(self): + value = self.module.normalize_published_at("not-a-date", now=self.now) + + self.assertEqual(value, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/sql/repair_article_future_published_at.sql b/docs/sql/repair_article_future_published_at.sql new file mode 100644 index 00000000..ee69665a --- /dev/null +++ b/docs/sql/repair_article_future_published_at.sql @@ -0,0 +1,8 @@ +-- 修复明显晚于文章入库时间的异常发布时间。 +-- 条件以创建时间后 2 天为容差，兼容数据源与数据库时区差异；重复执行不会再次命中。 +UPDATE `la_article` +SET `published_at` = DATE_FORMAT(FROM_UNIXTIME(create_time), '%Y-%m-%d %H:%i:%s') +WHERE `create_time` IS NOT NULL + AND `published_at` REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$' + AND STR_TO_DATE(`published_at`, '%Y-%m-%d %H:%i:%s') + > DATE_ADD(FROM_UNIXTIME(create_time), INTERVAL 2 DAY); diff --git a/qa/backend/test_article_publish_sort_static.php b/qa/backend/test_article_publish_sort_static.php new file mode 100644 index 00000000..9229dace --- /dev/null +++ b/qa/backend/test_article_publish_sort_static.php @@ -0,0 +1,30 @@ + str_contains($listsSource, 'buildSafePublishedOrder'), + '排序会解析标准发布时间' => str_contains($listsSource, "STR_TO_DATE(published_at, '%Y-%m-%d %H:%i:%s')"), + '排序拒绝超过两天的未来时间' => str_contains($listsSource, 'DATE_ADD(NOW(), INTERVAL 2 DAY)'), + '异常发布时间回退到创建时间' => str_contains($listsSource, 'FROM_UNIXTIME(create_time)'), + '采集入库前规范化发布时间' => str_contains($databaseSource, 'normalize_published_at'), + '重复采集允许刷新有效发布时间' => str_contains($databaseSource, 'published_at=IF(VALUES(published_at)'), + '提供幂等的历史未来时间修复 SQL' => str_contains($repairSource, 'DATE_ADD(FROM_UNIXTIME(create_time), INTERVAL 2 DAY)'), + '历史时间修复回退到创建时间' => str_contains($repairSource, 'DATE_FORMAT(FROM_UNIXTIME(create_time)'), +]; + +foreach ($checks as $label => $passed) { + if (!$passed) { + fwrite(STDERR, "FAIL: {$label}\n"); + exit(1); + } +} + +echo "文章发布时间排序与采集保护静态检查通过\n"; diff --git a/server/app/api/lists/article/ArticleLists.php b/server/app/api/lists/article/ArticleLists.php index 0b736f0a..8e8e2ebf 100644 --- a/server/app/api/lists/article/ArticleLists.php +++ b/server/app/api/lists/article/ArticleLists.php @@ -97,12 +97,12 @@ class ArticleLists extends BaseApiDataLists implements ListsSearchInterface */ public function lists(): array { - $orderRaw = 'published_at desc, id desc'; + $orderRaw = $this->buildSafePublishedOrder(); $sortType = $this->params['sort'] ?? 'default'; $cid = (int) ($this->params['cid'] ?? 0); // 最新排序 if ($sortType == 'new') { - $orderRaw = 'published_at desc, id desc'; + $orderRaw = $this->buildSafePublishedOrder(); } // 最热排序 if ($sortType == 'hot') { @@ -140,6 +140,20 @@ class ArticleLists extends BaseApiDataLists implements ListsSearchInterface } + /** + * 明显超前的脏发布时间不参与置顶排序，回退到文章入库时间。 + */ + private function buildSafePublishedOrder(): string + { + return "CASE + WHEN published_at REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$' + AND STR_TO_DATE(published_at, '%Y-%m-%d %H:%i:%s') <= DATE_ADD(NOW(), INTERVAL 2 DAY) + THEN STR_TO_DATE(published_at, '%Y-%m-%d %H:%i:%s') + ELSE FROM_UNIXTIME(create_time) + END DESC, id DESC"; + } + + /** * @notes 获取文章数量 * @return int