haha/sbnews
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore category news collection

Browse Source
This commit is contained in:
hajimi
2026-08-04 00:58:22 +08:00
parent bdab4f5019
commit edce4baa09
10 changed files with 100 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/crawler/crypto_news.py
+4 -4
View File
@@ -17,7 +17,7 @@ from email.utils import parsedate_to_datetime
import pymysql
import requests
from article_publish_helper import has_publishable_article_content, resolve_crawled_article_is_show
from article_publish_helper import resolve_crawled_article_is_show
# ─── 配置 ───
ARTICLE_CID = 9 # la_article_cate 中"加密资讯"的 id
@@ -187,7 +187,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
if source_url in existing:
existing_row = existing[source_url]
next_is_show = existing_row["is_show"] if has_publishable_article_content(existing_row.get("content", "")) else 0
next_is_show = resolve_crawled_article_is_show(existing_row.get("content", ""), allow_summary=True)
with conn.cursor() as cur:
cur.execute(
"""UPDATE la_article SET
@@ -230,7 +230,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
source_url[:500],
article.get("content") or "",
ext_json,
resolve_crawled_article_is_show(article.get("content") or ""),
resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True),
0,
0,
0,
@@ -238,7 +238,7 @@ def sync_to_db(conn, articles: list[dict]) -> tuple[int, int, int]:
now,
)
)
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "")}
existing[source_url] = {"id": cur.lastrowid, "content": article.get("content") or "", "is_show": resolve_crawled_article_is_show(article.get("content") or "", allow_summary=True)}
existing_titles.add(article["title"][:255])
new_count += 1