fix: restore category news collection

docker/crawler/article_fetcher.py

@@ -24,7 +24,7 @@ from scrapling.fetchers import Fetcher
 TARGET_CIDS = [9, 17, 18, 21]  # 需要抓正文的分类
 BATCH_SIZE = 3                  # 每个分类每次抓取条数
 MAX_WORKERS = 4                 # 并行抓取线程数
-MAX_RETRY = 2                   # 最大重试次数，达到后永久跳过
+MAX_RETRY = 4                   # 最大重试次数，达到后暂时跳过
 REQUESTS_FALLBACK_HEADERS = {
     "User-Agent": (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -389,9 +389,15 @@ def run(db_config: dict):
         with conn.cursor() as cur:
             for cid in TARGET_CIDS:
                 cur.execute(
-                    """SELECT id, title, source_url, content, is_show FROM la_article
+                    """SELECT id, title, source_url, content FROM la_article
                     WHERE cid = %s
-                      AND ((content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50) OR is_show = 0)
+                      AND (
+                          content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50
+                          OR (
+                              CHAR_LENGTH(content) < 200
+                              AND content NOT REGEXP '<(p|div|article|section|h[1-6]|blockquote|ul|ol|li|pre|table|figure)([[:space:]>])'
+                          )
+                      )
                       AND content_retry < %s
                       AND source_url IS NOT NULL AND source_url != ''
                     ORDER BY content_retry ASC, id DESC