diff --git a/docker/crawler/article_fetcher.py b/docker/crawler/article_fetcher.py index 0cdeaf00..6fb3993d 100644 --- a/docker/crawler/article_fetcher.py +++ b/docker/crawler/article_fetcher.py @@ -24,7 +24,7 @@ from scrapling.fetchers import Fetcher TARGET_CIDS = [9, 17, 18, 21] # 需要抓正文的分类 BATCH_SIZE = 3 # 每个分类每次抓取条数 MAX_WORKERS = 4 # 并行抓取线程数 -MAX_RETRY = 2 # 最大重试次数，达到后永久跳过 +MAX_RETRY = 4 # 最大重试次数，达到后暂时跳过 REQUESTS_FALLBACK_HEADERS = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " @@ -389,9 +389,15 @@ def run(db_config: dict): with conn.cursor() as cur: for cid in TARGET_CIDS: cur.execute( - """SELECT id, title, source_url, content, is_show FROM la_article + """SELECT id, title, source_url, content FROM la_article WHERE cid = %s - AND ((content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50) OR is_show = 0) + AND ( + content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50 + OR ( + CHAR_LENGTH(content) < 200 + AND content NOT REGEXP '<(p|div|article|section|h[1-6]|blockquote|ul|ol|li|pre|table|figure)([[:space:]>])' + ) + ) AND content_retry < %s AND source_url IS NOT NULL AND source_url != '' ORDER BY content_retry ASC, id DESC diff --git a/docker/crawler/article_publish_helper.py b/docker/crawler/article_publish_helper.py index 2642277d..9e7d4995 100644 --- a/docker/crawler/article_publish_helper.py +++ b/docker/crawler/article_publish_helper.py @@ -22,6 +22,8 @@ def has_publishable_article_content(content: str) -> bool: return len(plain) >= 200 -def resolve_crawled_article_is_show(content: str) -> int: +def resolve_crawled_article_is_show(content: str, allow_summary: bool = False) -> int: """根据正文内容决定采集文章的默认状态：1=发布，0=稿子。""" + if allow_summary and (content or "").strip(): + return 1 return 1 if has_publishable_article_content(content) else 0 diff --git a/docker/crawler/cba_news.py b/docker/crawler/cba_news.py index 125547c5..5cb463e0 100644 --- a/docker/crawler/cba_news.py +++ b/docker/crawler/cba_news.py @@ -17,7 +17,7 @@ from email.utils import parsedate_to_datetime import pymysql import requests -from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show +from article_publish_helper import resolve_crawled_article_is_show # ─── 配置 ─── ARTICLE_CID = 18 # la_article_cate 中"CBA"的 id @@ -185,7 +185,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: if source_url in existing: existing_row = existing[source_url] - next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0 + next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True) with conn.cursor() as cur: cur.execute( """UPDATE la_article SET @@ -228,7 +228,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: source_url[:500], article.get("content") or "", ext_json, - resolve_crawled_article_is_show(article.get("content") or ""), + resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True), 0, 0, 0, @@ -236,7 +236,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: now, ) ) - existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")} + existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)} existing_titles.add(article["title"][:255]) new_count += 1 diff --git a/docker/crawler/config/crawler_tasks.yaml b/docker/crawler/config/crawler_tasks.yaml index 294fe14d..83f094d1 100644 --- a/docker/crawler/config/crawler_tasks.yaml +++ b/docker/crawler/config/crawler_tasks.yaml @@ -129,6 +129,11 @@ tasks: action: lottery_community cron: "*/10 * * * *" active: true + - key: lottery_news + name: 彩票新闻采集 + action: lottery_news + cron: "30 */2 * * *" + active: true - key: taiwan_lottery_news name: 台湾彩券资讯采集 action: taiwan_lottery_news diff --git a/docker/crawler/crypto_news.py b/docker/crawler/crypto_news.py index 2d30863b..df976c89 100644 --- a/docker/crawler/crypto_news.py +++ b/docker/crawler/crypto_news.py @@ -17,7 +17,7 @@ from email.utils import parsedate_to_datetime import pymysql import requests -from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show +from article_publish_helper import resolve_crawled_article_is_show # ─── 配置 ─── ARTICLE_CID = 9 # la_article_cate 中"加密资讯"的 id @@ -187,7 +187,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: if source_url in existing: existing_row = existing[source_url] - next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0 + next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True) with conn.cursor() as cur: cur.execute( """UPDATE la_article SET @@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: source_url[:500], article.get("content") or "", ext_json, - resolve_crawled_article_is_show(article.get("content") or ""), + resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True), 0, 0, 0, @@ -238,7 +238,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: now, ) ) - existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")} + existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)} existing_titles.add(article["title"][:255]) new_count += 1 diff --git a/docker/crawler/hkjc_lottery_news.py b/docker/crawler/hkjc_lottery_news.py index be727c3d..4d8fd511 100644 --- a/docker/crawler/hkjc_lottery_news.py +++ b/docker/crawler/hkjc_lottery_news.py @@ -1,6 +1,6 @@ """ 香港赛马会新闻采集模块 -- 列表页: https://racingnews.hkjc.com/chinese/?ny=2026&nm=6 +- 列表页按当前年月动态请求 - 详情页: 列表内详情链接 写入分类: @@ -18,7 +18,7 @@ import requests from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show ARTICLE_CID = 21 -LIST_URL = "https://racingnews.hkjc.com/chinese/?ny=2026&nm=6" +LIST_URL_TEMPLATE = "https://racingnews.hkjc.com/chinese/?ny={year}&nm={month:02d}" SOURCE_NAME = "香港赛马会" MAX_ITEMS = 20 @@ -27,6 +27,11 @@ def clean_text(text: str) -> str: return re.sub(r"\s+", " ", html.unescape(text or "").replace("\u3000", " ")).strip() +def build_list_url(now: datetime | None = None) -> str: + current = now or datetime.now() + return LIST_URL_TEMPLATE.format(year=current.year, month=current.month) + + def parse_hk_date(date_str: str) -> tuple[int, str]: if not date_str: now = int(time.time()) @@ -42,8 +47,9 @@ def parse_hk_date(date_str: str) -> tuple[int, str]: return now, datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S") -def fetch_list(session: requests.Session) -> list[dict]: - resp = session.get(LIST_URL, timeout=30) +def fetch_list(session: requests.Session, now: datetime | None = None) -> list[dict]: + list_url = build_list_url(now) + resp = session.get(list_url, timeout=30) resp.raise_for_status() html_text = resp.text matches = re.findall( @@ -56,7 +62,7 @@ def fetch_list(session: requests.Session) -> list[dict]: for date_text, href, title in matches[:MAX_ITEMS]: items.append({ "date_text": clean_text(re.sub(r"<[^>]+>", " ", date_text)), - "source_url": urljoin(LIST_URL, href), + "source_url": urljoin(list_url, href), "title": clean_text(re.sub(r"<[^>]+>", " ", title)), }) return items @@ -142,7 +148,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: ext_json = json.dumps({ "source": "hkjc_racingnews", - "list_url": LIST_URL, + "list_url": build_list_url(), }, ensure_ascii=False) if source_url in existing: diff --git a/docker/crawler/lottery_news.py b/docker/crawler/lottery_news.py index 7f0af91c..a3557a64 100644 --- a/docker/crawler/lottery_news.py +++ b/docker/crawler/lottery_news.py @@ -16,7 +16,7 @@ from email.utils import parsedate_to_datetime import pymysql import requests -from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show +from article_publish_helper import resolve_crawled_article_is_show # ─── 配置 ─── ARTICLE_CID = 21 # la_article_cate 中"彩票资讯"的 id @@ -179,7 +179,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: if source_url in existing: existing_row = existing[source_url] - next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0 + next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True) with conn.cursor() as cur: cur.execute( """UPDATE la_article SET @@ -222,7 +222,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: source_url[:500], article.get("content") or "", ext_json, - resolve_crawled_article_is_show(article.get("content") or ""), + resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True), 0, 0, 0, @@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: now, ) ) - existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")} + existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)} existing_titles.add(article["title"][:255]) new_count += 1 diff --git a/docker/crawler/nba_news.py b/docker/crawler/nba_news.py index 31665546..beb859e7 100644 --- a/docker/crawler/nba_news.py +++ b/docker/crawler/nba_news.py @@ -16,7 +16,7 @@ from email.utils import parsedate_to_datetime import pymysql import requests -from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show +from article_publish_helper import resolve_crawled_article_is_show # ─── 配置 ─── ARTICLE_CID = 17 # la_article_cate 中"NBA"的 id @@ -179,7 +179,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: if source_url in existing: existing_row = existing[source_url] - next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0 + next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True) with conn.cursor() as cur: cur.execute( """UPDATE la_article SET @@ -222,7 +222,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: source_url[:500], article.get("content") or "", ext_json, - resolve_crawled_article_is_show(article.get("content") or ""), + resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True), 0, 0, 0, @@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]: now, ) ) - existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")} + existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)} existing_titles.add(article["title"][:255]) new_count += 1 diff --git a/docker/crawler/taiwan_lottery_news.py b/docker/crawler/taiwan_lottery_news.py index 178def2d..79d98e65 100644 --- a/docker/crawler/taiwan_lottery_news.py +++ b/docker/crawler/taiwan_lottery_news.py @@ -1,6 +1,6 @@ """ 台湾彩券资讯采集模块 -- 列表接口: https://api.taiwanlottery.com/TLCAPIWeB/News/List +- 列表接口按当前年月动态请求 - 详情页面: https://www.taiwanlottery.com/news/news/{newsId} 写入分类: @@ -18,7 +18,7 @@ import requests from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show ARTICLE_CID = 21 -LIST_API = "https://api.taiwanlottery.com/TLCAPIWeB/News/List?keyword&from=2025-06&to=2026-06&type=2&webtag=1&pageSize=20&pageNo={page}" +LIST_API_TEMPLATE = "https://api.taiwanlottery.com/TLCAPIWeB/News/List?keyword&from={from_month}&to={to_month}&type=2&webtag=1&pageSize=20&pageNo={page}" DETAIL_URL = "https://www.taiwanlottery.com/news/news/{news_id}" DETAIL_API = "https://api.taiwanlottery.com/TLCAPIWeB/News/Detail/{news_id}" SOURCE_NAME = "台湾彩券" @@ -29,6 +29,13 @@ def clean_text(text: str) -> str: return re.sub(r"\s+", " ", html.unescape(text or "").replace("\u3000", " ")).strip() +def build_list_api(page_no: int, now: datetime | None = None) -> str: + current = now or datetime.now() + from_month = f"{current.year - 1:04d}-{current.month:02d}" + to_month = f"{current.year:04d}-{current.month:02d}" + return LIST_API_TEMPLATE.format(from_month=from_month, to_month=to_month, page=page_no) + + def parse_announce_date(value: str) -> tuple[int, str]: if not value: now = int(time.time()) @@ -42,8 +49,8 @@ def parse_announce_date(value: str) -> tuple[int, str]: return now, datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S") -def fetch_list(session: requests.Session, page_no: int) -> list[dict]: - resp = session.get(LIST_API.format(page=page_no), timeout=30) +def fetch_list(session: requests.Session, page_no: int, now: datetime | None = None) -> list[dict]: + resp = session.get(build_list_api(page_no, now), timeout=30) resp.raise_for_status() data = resp.json() if data.get("rtCode") != 0: diff --git a/qa/crawler/test_news_collection_static.py b/qa/crawler/test_news_collection_static.py new file mode 100644 index 00000000..712dfe8f --- /dev/null +++ b/qa/crawler/test_news_collection_static.py @@ -0,0 +1,44 @@ +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +CRAWLER = ROOT / "docker" / "crawler" + + +class NewsCollectionRegressionTest(unittest.TestCase): + def read(self, name: str) -> str: + return (CRAWLER / name).read_text(encoding="utf-8") + + def test_lottery_sources_use_runtime_month(self): + hkjc = self.read("hkjc_lottery_news.py") + taiwan = self.read("taiwan_lottery_news.py") + + self.assertNotIn("?ny=2026&nm=6", hkjc) + self.assertIn("LIST_URL_TEMPLATE", hkjc) + self.assertIn("def build_list_url", hkjc) + self.assertNotIn("from=2025-06&to=2026-06", taiwan) + self.assertIn("LIST_API_TEMPLATE", taiwan) + self.assertIn("def build_list_api", taiwan) + + def test_general_lottery_task_is_registered(self): + tasks = (ROOT / "docker" / "crawler" / "config" / "crawler_tasks.yaml").read_text(encoding="utf-8") + block_start = tasks.index(" - key: lottery_news") + block = tasks[block_start : tasks.find("\n - key:", block_start + 1)] + self.assertIn("action: lottery_news", block) + self.assertIn("active: true", block) + + def test_content_queue_filters_non_publishable_body_not_visibility(self): + source = self.read("article_fetcher.py") + self.assertNotIn("OR is_show = 0", source) + self.assertIn("NOT REGEXP", source) + self.assertIn("MAX_RETRY = 4", source) + + def test_rss_categories_allow_summary_to_be_listed(self): + for name in ("lottery_news.py", "crypto_news.py", "nba_news.py", "cba_news.py"): + source = self.read(name) + self.assertIn("allow_summary=True", source, name) + + +if __name__ == "__main__": + unittest.main()