fix: restore category news collection

docker/crawler/article_fetcher.py

@@ -24,7 +24,7 @@ from scrapling.fetchers import Fetcher
 TARGET_CIDS = [9, 17, 18, 21]  # 需要抓正文的分类
 BATCH_SIZE = 3                  # 每个分类每次抓取条数
 MAX_WORKERS = 4                 # 并行抓取线程数
-MAX_RETRY = 2                   # 最大重试次数，达到后永久跳过
+MAX_RETRY = 4                   # 最大重试次数，达到后暂时跳过
 REQUESTS_FALLBACK_HEADERS = {
     "User-Agent": (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -389,9 +389,15 @@ def run(db_config: dict):
         with conn.cursor() as cur:
             for cid in TARGET_CIDS:
                 cur.execute(
-                    """SELECT id, title, source_url, content, is_show FROM la_article
+                    """SELECT id, title, source_url, content FROM la_article
                     WHERE cid = %s
-                      AND ((content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50) OR is_show = 0)
+                      AND (
+                          content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50
+                          OR (
+                              CHAR_LENGTH(content) < 200
+                              AND content NOT REGEXP '<(p|div|article|section|h[1-6]|blockquote|ul|ol|li|pre|table|figure)([[:space:]>])'
+                          )
+                      )
                       AND content_retry < %s
                       AND source_url IS NOT NULL AND source_url != ''
                     ORDER BY content_retry ASC, id DESC

docker/crawler/article_publish_helper.py

@@ -22,6 +22,8 @@ def has_publishable_article_content(content: str) -> bool:
     return len(plain) >= 200
 
 
-def resolve_crawled_article_is_show(content: str) -> int:
+def resolve_crawled_article_is_show(content: str, allow_summary: bool = False) -> int:
     """根据正文内容决定采集文章的默认状态：1=发布，0=稿子。"""
+    if allow_summary and (content or "").strip():
+        return 1
     return 1 if has_publishable_article_content(content) else 0

docker/crawler/cba_news.py

@@ -17,7 +17,7 @@ from email.utils import parsedate_to_datetime
 
 import pymysql
 import requests
-from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
+from article_publish_helper import resolve_crawled_article_is_show
 
 # ─── 配置 ───
 ARTICLE_CID = 18  # la_article_cate 中"CBA"的 id
@@ -185,7 +185,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
 
         if source_url in existing:
             existing_row = existing[source_url]
-            next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
+            next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
             with conn.cursor() as cur:
                 cur.execute(
                     """UPDATE la_article SET
@@ -228,7 +228,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         source_url[:500],
                         article.get("content") or "",
                         ext_json,
-                        resolve_crawled_article_is_show(article.get("content") or ""),
+                        resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
                         0,
                         0,
                         0,
@@ -236,7 +236,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         now,
                     )
                 )
-            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
+            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
             existing_titles.add(article["title"][:255])
             new_count += 1
 

docker/crawler/config/crawler_tasks.yaml

@@ -129,6 +129,11 @@ tasks:
     action: lottery_community
     cron: "*/10 * * * *"
     active: true
+  - key: lottery_news
+    name: 彩票新闻采集
+    action: lottery_news
+    cron: "30 */2 * * *"
+    active: true
   - key: taiwan_lottery_news
     name: 台湾彩券资讯采集
     action: taiwan_lottery_news

docker/crawler/crypto_news.py

@@ -17,7 +17,7 @@ from email.utils import parsedate_to_datetime
 
 import pymysql
 import requests
-from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
+from article_publish_helper import resolve_crawled_article_is_show
 
 # ─── 配置 ───
 ARTICLE_CID = 9  # la_article_cate 中"加密资讯"的 id
@@ -187,7 +187,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
 
         if source_url in existing:
             existing_row = existing[source_url]
-            next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
+            next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
             with conn.cursor() as cur:
                 cur.execute(
                     """UPDATE la_article SET
@@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         source_url[:500],
                         article.get("content") or "",
                         ext_json,
-                        resolve_crawled_article_is_show(article.get("content") or ""),
+                        resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
                         0,
                         0,
                         0,
@@ -238,7 +238,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         now,
                     )
                 )
-            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
+            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
             existing_titles.add(article["title"][:255])
             new_count += 1
 

docker/crawler/hkjc_lottery_news.py

@@ -1,6 +1,6 @@
 """
 香港赛马会新闻采集模块
-- 列表页: https://racingnews.hkjc.com/chinese/?ny=2026&nm=6
+- 列表页按当前年月动态请求
 - 详情页: 列表内详情链接
 
 写入分类:
@@ -18,7 +18,7 @@ import requests
 from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
 
 ARTICLE_CID = 21
-LIST_URL = "https://racingnews.hkjc.com/chinese/?ny=2026&nm=6"
+LIST_URL_TEMPLATE = "https://racingnews.hkjc.com/chinese/?ny={year}&nm={month:02d}"
 SOURCE_NAME = "香港赛马会"
 MAX_ITEMS = 20
 
@@ -27,6 +27,11 @@ def clean_text(text: str) -> str:
     return re.sub(r"\s+", " ", html.unescape(text or "").replace("\u3000", " ")).strip()
 
 
+def build_list_url(now: datetime | None = None) -> str:
+    current = now or datetime.now()
+    return LIST_URL_TEMPLATE.format(year=current.year, month=current.month)
+
+
 def parse_hk_date(date_str: str) -> tuple[int, str]:
     if not date_str:
         now = int(time.time())
@@ -42,8 +47,9 @@ def parse_hk_date(date_str: str) -> tuple[int, str]:
     return now, datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S")
 
 
-def fetch_list(session: requests.Session) -> list[dict]:
-    resp = session.get(LIST_URL, timeout=30)
+def fetch_list(session: requests.Session, now: datetime | None = None) -> list[dict]:
+    list_url = build_list_url(now)
+    resp = session.get(list_url, timeout=30)
     resp.raise_for_status()
     html_text = resp.text
     matches = re.findall(
@@ -56,7 +62,7 @@ def fetch_list(session: requests.Session) -> list[dict]:
     for date_text, href, title in matches[:MAX_ITEMS]:
         items.append({
             "date_text": clean_text(re.sub(r"<[^>]+>", " ", date_text)),
-            "source_url": urljoin(LIST_URL, href),
+            "source_url": urljoin(list_url, href),
             "title": clean_text(re.sub(r"<[^>]+>", " ", title)),
         })
     return items
@@ -142,7 +148,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
 
         ext_json = json.dumps({
             "source": "hkjc_racingnews",
-            "list_url": LIST_URL,
+            "list_url": build_list_url(),
         }, ensure_ascii=False)
 
         if source_url in existing:

docker/crawler/lottery_news.py

@@ -16,7 +16,7 @@ from email.utils import parsedate_to_datetime
 
 import pymysql
 import requests
-from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
+from article_publish_helper import resolve_crawled_article_is_show
 
 # ─── 配置 ───
 ARTICLE_CID = 21  # la_article_cate 中"彩票资讯"的 id
@@ -179,7 +179,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
 
         if source_url in existing:
             existing_row = existing[source_url]
-            next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
+            next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
             with conn.cursor() as cur:
                 cur.execute(
                     """UPDATE la_article SET
@@ -222,7 +222,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         source_url[:500],
                         article.get("content") or "",
                         ext_json,
-                        resolve_crawled_article_is_show(article.get("content") or ""),
+                        resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
                         0,
                         0,
                         0,
@@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         now,
                     )
                 )
-            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
+            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
             existing_titles.add(article["title"][:255])
             new_count += 1
 

docker/crawler/nba_news.py

@@ -16,7 +16,7 @@ from email.utils import parsedate_to_datetime
 
 import pymysql
 import requests
-from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
+from article_publish_helper import resolve_crawled_article_is_show
 
 # ─── 配置 ───
 ARTICLE_CID = 17  # la_article_cate 中"NBA"的 id
@@ -179,7 +179,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
 
         if source_url in existing:
             existing_row = existing[source_url]
-            next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
+            next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
             with conn.cursor() as cur:
                 cur.execute(
                     """UPDATE la_article SET
@@ -222,7 +222,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         source_url[:500],
                         article.get("content") or "",
                         ext_json,
-                        resolve_crawled_article_is_show(article.get("content") or ""),
+                        resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
                         0,
                         0,
                         0,
@@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
                         now,
                     )
                 )
-            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
+            existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
             existing_titles.add(article["title"][:255])
             new_count += 1
 

docker/crawler/taiwan_lottery_news.py

@@ -1,6 +1,6 @@
 """
 台湾彩券资讯采集模块
-- 列表接口: https://api.taiwanlottery.com/TLCAPIWeB/News/List
+- 列表接口按当前年月动态请求
 - 详情页面: https://www.taiwanlottery.com/news/news/{newsId}
 
 写入分类:
@@ -18,7 +18,7 @@ import requests
 from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
 
 ARTICLE_CID = 21
-LIST_API = "https://api.taiwanlottery.com/TLCAPIWeB/News/List?keyword&from=2025-06&to=2026-06&type=2&webtag=1&pageSize=20&pageNo={page}"
+LIST_API_TEMPLATE = "https://api.taiwanlottery.com/TLCAPIWeB/News/List?keyword&from={from_month}&to={to_month}&type=2&webtag=1&pageSize=20&pageNo={page}"
 DETAIL_URL = "https://www.taiwanlottery.com/news/news/{news_id}"
 DETAIL_API = "https://api.taiwanlottery.com/TLCAPIWeB/News/Detail/{news_id}"
 SOURCE_NAME = "台湾彩券"
@@ -29,6 +29,13 @@ def clean_text(text: str) -> str:
     return re.sub(r"\s+", " ", html.unescape(text or "").replace("\u3000", " ")).strip()
 
 
+def build_list_api(page_no: int, now: datetime | None = None) -> str:
+    current = now or datetime.now()
+    from_month = f"{current.year - 1:04d}-{current.month:02d}"
+    to_month = f"{current.year:04d}-{current.month:02d}"
+    return LIST_API_TEMPLATE.format(from_month=from_month, to_month=to_month, page=page_no)
+
+
 def parse_announce_date(value: str) -> tuple[int, str]:
     if not value:
         now = int(time.time())
@@ -42,8 +49,8 @@ def parse_announce_date(value: str) -> tuple[int, str]:
         return now, datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S")
 
 
-def fetch_list(session: requests.Session, page_no: int) -> list[dict]:
-    resp = session.get(LIST_API.format(page=page_no), timeout=30)
+def fetch_list(session: requests.Session, page_no: int, now: datetime | None = None) -> list[dict]:
+    resp = session.get(build_list_api(page_no, now), timeout=30)
     resp.raise_for_status()
     data = resp.json()
     if data.get("rtCode") != 0:

qa/crawler/test_news_collection_static.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+CRAWLER = ROOT / "docker" / "crawler"
+
+
+class NewsCollectionRegressionTest(unittest.TestCase):
+    def read(self, name: str) -> str:
+        return (CRAWLER / name).read_text(encoding="utf-8")
+
+    def test_lottery_sources_use_runtime_month(self):
+        hkjc = self.read("hkjc_lottery_news.py")
+        taiwan = self.read("taiwan_lottery_news.py")
+
+        self.assertNotIn("?ny=2026&nm=6", hkjc)
+        self.assertIn("LIST_URL_TEMPLATE", hkjc)
+        self.assertIn("def build_list_url", hkjc)
+        self.assertNotIn("from=2025-06&to=2026-06", taiwan)
+        self.assertIn("LIST_API_TEMPLATE", taiwan)
+        self.assertIn("def build_list_api", taiwan)
+
+    def test_general_lottery_task_is_registered(self):
+        tasks = (ROOT / "docker" / "crawler" / "config" / "crawler_tasks.yaml").read_text(encoding="utf-8")
+        block_start = tasks.index("  - key: lottery_news")
+        block = tasks[block_start : tasks.find("\n  - key:", block_start + 1)]
+        self.assertIn("action: lottery_news", block)
+        self.assertIn("active: true", block)
+
+    def test_content_queue_filters_non_publishable_body_not_visibility(self):
+        source = self.read("article_fetcher.py")
+        self.assertNotIn("OR is_show = 0", source)
+        self.assertIn("NOT REGEXP", source)
+        self.assertIn("MAX_RETRY = 4", source)
+
+    def test_rss_categories_allow_summary_to_be_listed(self):
+        for name in ("lottery_news.py", "crypto_news.py", "nba_news.py", "cba_news.py"):
+            source = self.read(name)
+            self.assertIn("allow_summary=True", source, name)
+
+
+if __name__ == "__main__":
+    unittest.main()