haha/sbnews
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore category news collection

Browse Source
This commit is contained in:
hajimi
2026-08-04 00:58:22 +08:00
parent bdab4f5019
commit edce4baa09
10 changed files with 100 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/crawler/article_fetcher.py
+9 -3
View File
@@ -24,7 +24,7 @@ from scrapling.fetchers import Fetcher
TARGET_CIDS = [9, 17, 18, 21] # 需要抓正文的分类
BATCH_SIZE = 3 # 每个分类每次抓取条数
MAX_WORKERS = 4 # 并行抓取线程数
MAX_RETRY = 2 # 最大重试次数，达到后永久跳过
MAX_RETRY = 4 # 最大重试次数，达到后暂时跳过
REQUESTS_FALLBACK_HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -389,9 +389,15 @@ def run(db_config: dict):
with conn.cursor() as cur:
for cid in TARGET_CIDS:
cur.execute(
"""SELECT id, title, source_url, content, is_show FROM la_article
"""SELECT id, title, source_url, content FROM la_article
WHERE cid = %s
AND ((content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50) OR is_show = 0)
AND (
content IS NULL OR content = '' OR CHAR_LENGTH(content) < 50
OR (
CHAR_LENGTH(content) < 200
AND content NOT REGEXP '<(p|div|article|section|h[1-6]|blockquote|ul|ol|li|pre|table|figure)([[:space:]>])'
)
)
AND content_retry < %s
AND source_url IS NOT NULL AND source_url != ''
ORDER BY content_retry ASC, id DESC
docker/crawler/article_publish_helper.py
+3 -1
View File
@@ -22,6 +22,8 @@ def has_publishable_article_content(content: str) -> bool:
return len(plain) >= 200
def resolve_crawled_article_is_show(content: str) -> int:
def resolve_crawled_article_is_show(content: str, allow_summary: bool = False) -> int:
"""根据正文内容决定采集文章的默认状态：1=发布，0=稿子。"""
if allow_summary and (content or "").strip():
return 1
return 1 if has_publishable_article_content(content) else 0
docker/crawler/cba_news.py
+4 -4
View File
@@ -17,7 +17,7 @@ from email.utils import parsedate_to_datetime
import pymysql
import requests
from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
from article_publish_helper import resolve_crawled_article_is_show
# ─── 配置 ───
ARTICLE_CID = 18 # la_article_cate 中"CBA"的 id
@@ -185,7 +185,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
if source_url in existing:
existing_row = existing[source_url]
next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
with conn.cursor() as cur:
cur.execute(
"""UPDATE la_article SET
@@ -228,7 +228,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
source_url[:500],
article.get("content") or "",
ext_json,
resolve_crawled_article_is_show(article.get("content") or ""),
resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
0,
0,
0,
@@ -236,7 +236,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
now,
)
)
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
existing_titles.add(article["title"][:255])
new_count += 1
docker/crawler/config/crawler_tasks.yaml
+5
View File
@@ -129,6 +129,11 @@ tasks:
action: lottery_community
cron: "*/10 * * * *"
active: true
- key: lottery_news
name: 彩票新闻采集
action: lottery_news
cron: "30 */2 * * *"
active: true
- key: taiwan_lottery_news
name: 台湾彩券资讯采集
action: taiwan_lottery_news
docker/crawler/crypto_news.py
+4 -4
View File
@@ -17,7 +17,7 @@ from email.utils import parsedate_to_datetime
import pymysql
import requests
from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
from article_publish_helper import resolve_crawled_article_is_show
# ─── 配置 ───
ARTICLE_CID = 9 # la_article_cate 中"加密资讯"的 id
@@ -187,7 +187,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
if source_url in existing:
existing_row = existing[source_url]
next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
with conn.cursor() as cur:
cur.execute(
"""UPDATE la_article SET
@@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
source_url[:500],
article.get("content") or "",
ext_json,
resolve_crawled_article_is_show(article.get("content") or ""),
resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
0,
0,
0,
@@ -238,7 +238,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
now,
)
)
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
existing_titles.add(article["title"][:255])
new_count += 1
docker/crawler/hkjc_lottery_news.py
+12 -6
View File
@@ -1,6 +1,6 @@
"""
香港赛马会新闻采集模块
- 列表页: https://racingnews.hkjc.com/chinese/?ny=2026&nm=6
- 列表页按当前年月动态请求
- 详情页: 列表内详情链接
写入分类:
@@ -18,7 +18,7 @@ import requests
from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
ARTICLE_CID = 21
LIST_URL = "https://racingnews.hkjc.com/chinese/?ny=2026&nm=6"
LIST_URL_TEMPLATE = "https://racingnews.hkjc.com/chinese/?ny={year}&nm={month:02d}"
SOURCE_NAME = "香港赛马会"
MAX_ITEMS = 20
@@ -27,6 +27,11 @@ def clean_text(text: str) -> str:
return re.sub(r"\s+", " ", html.unescape(text or "").replace("\u3000", " ")).strip()
def build_list_url(now: datetime | None = None) -> str:
current = now or datetime.now()
return LIST_URL_TEMPLATE.format(year=current.year, month=current.month)
def parse_hk_date(date_str: str) -> tuple[int, str]:
if not date_str:
now = int(time.time())
@@ -42,8 +47,9 @@ def parse_hk_date(date_str: str) -> tuple[int, str]:
return now, datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S")
def fetch_list(session: requests.Session) -> list[dict]:
resp = session.get(LIST_URL, timeout=30)
def fetch_list(session: requests.Session, now: datetime | None = None) -> list[dict]:
list_url = build_list_url(now)
resp = session.get(list_url, timeout=30)
resp.raise_for_status()
html_text = resp.text
matches = re.findall(
@@ -56,7 +62,7 @@ def fetch_list(session: requests.Session) -> list[dict]:
for date_text, href, title in matches[:MAX_ITEMS]:
items.append({
"date_text": clean_text(re.sub(r"<[^>]+>", " ", date_text)),
"source_url": urljoin(LIST_URL, href),
"source_url": urljoin(list_url, href),
"title": clean_text(re.sub(r"<[^>]+>", " ", title)),
})
return items
@@ -142,7 +148,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
ext_json = json.dumps({
"source": "hkjc_racingnews",
"list_url": LIST_URL,
"list_url": build_list_url(),
}, ensure_ascii=False)
if source_url in existing:
docker/crawler/lottery_news.py
+4 -4
View File
@@ -16,7 +16,7 @@ from email.utils import parsedate_to_datetime
import pymysql
import requests
from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
from article_publish_helper import resolve_crawled_article_is_show
# ─── 配置 ───
ARTICLE_CID = 21 # la_article_cate 中"彩票资讯"的 id
@@ -179,7 +179,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
if source_url in existing:
existing_row = existing[source_url]
next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
with conn.cursor() as cur:
cur.execute(
"""UPDATE la_article SET
@@ -222,7 +222,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
source_url[:500],
article.get("content") or "",
ext_json,
resolve_crawled_article_is_show(article.get("content") or ""),
resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
0,
0,
0,
@@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
now,
)
)
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
existing_titles.add(article["title"][:255])
new_count += 1
docker/crawler/nba_news.py
+4 -4
View File
@@ -16,7 +16,7 @@ from email.utils import parsedate_to_datetime
import pymysql
import requests
from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
from article_publish_helper import resolve_crawled_article_is_show
# ─── 配置 ───
ARTICLE_CID = 17 # la_article_cate 中"NBA"的 id
@@ -179,7 +179,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
if source_url in existing:
existing_row = existing[source_url]
next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
with conn.cursor() as cur:
cur.execute(
"""UPDATE la_article SET
@@ -222,7 +222,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
source_url[:500],
article.get("content") or "",
ext_json,
resolve_crawled_article_is_show(article.get("content") or ""),
resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
0,
0,
0,
@@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
now,
)
)
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
existing_titles.add(article["title"][:255])
new_count += 1
docker/crawler/taiwan_lottery_news.py
+11 -4
View File
@@ -1,6 +1,6 @@
"""
台湾彩券资讯采集模块
- 列表接口: https://api.taiwanlottery.com/TLCAPIWeB/News/List
- 列表接口按当前年月动态请求
- 详情页面: https://www.taiwanlottery.com/news/news/{newsId}
写入分类:
@@ -18,7 +18,7 @@ import requests
from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
ARTICLE_CID = 21
LIST_API = "https://api.taiwanlottery.com/TLCAPIWeB/News/List?keyword&from=2025-06&to=2026-06&type=2&webtag=1&pageSize=20&pageNo={page}"
LIST_API_TEMPLATE = "https://api.taiwanlottery.com/TLCAPIWeB/News/List?keyword&from={from_month}&to={to_month}&type=2&webtag=1&pageSize=20&pageNo={page}"
DETAIL_URL = "https://www.taiwanlottery.com/news/news/{news_id}"
DETAIL_API = "https://api.taiwanlottery.com/TLCAPIWeB/News/Detail/{news_id}"
SOURCE_NAME = "台湾彩券"
@@ -29,6 +29,13 @@ def clean_text(text: str) -> str:
return re.sub(r"\s+", " ", html.unescape(text or "").replace("\u3000", " ")).strip()
def build_list_api(page_no: int, now: datetime | None = None) -> str:
current = now or datetime.now()
from_month = f"{current.year - 1:04d}-{current.month:02d}"
to_month = f"{current.year:04d}-{current.month:02d}"
return LIST_API_TEMPLATE.format(from_month=from_month, to_month=to_month, page=page_no)
def parse_announce_date(value: str) -> tuple[int, str]:
if not value:
now = int(time.time())
@@ -42,8 +49,8 @@ def parse_announce_date(value: str) -> tuple[int, str]:
return now, datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S")
def fetch_list(session: requests.Session, page_no: int) -> list[dict]:
resp = session.get(LIST_API.format(page=page_no), timeout=30)
def fetch_list(session: requests.Session, page_no: int, now: datetime | None = None) -> list[dict]:
resp = session.get(build_list_api(page_no, now), timeout=30)
resp.raise_for_status()
data = resp.json()
if data.get("rtCode") != 0: