迁移目录

2026-06-12 22:27:18 +08:00
parent 7381097582
commit cd44cd6e47
92 changed files with 1839 additions and 0 deletions
@@ -0,0 +1,135 @@
+"""
+懂球帝文章正文采集脚本
+- 从数据库读取已入库但无正文的文章
+- 请求文章详情页HTML，提取 div.con 正文内容
+- 将正文HTML写入 la_article.content
+用法:
+  python scripts/crawl_article_detail.py              # 采集最多50篇
+  python scripts/crawl_article_detail.py --limit 200  # 采集最多200篇
+"""
+import asyncio
+import random
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from bs4 import BeautifulSoup
+from curl_cffi import requests as curl_requests
+from loguru import logger
+
+from src.core.config import get_config
+from src.storage.database import Database
+
+DETAIL_URL = "https://www.dongqiudi.com/articles/{article_id}"
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                  "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Referer": "https://www.dongqiudi.com/",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+}
+
+
+def fetch_article_html(article_id: int) -> str:
+    url = DETAIL_URL.format(article_id=article_id)
+    resp = curl_requests.get(
+        url, headers=HEADERS,
+        impersonate="chrome", timeout=20,
+    )
+    resp.raise_for_status()
+    return resp.text
+
+
+def extract_content(html: str) -> str:
+    soup = BeautifulSoup(html, "html.parser")
+    con_div = soup.select_one("div.con")
+    if not con_div:
+        return ""
+    inner = con_div.find("div", style=lambda v: v and "display:none" in v.replace(" ", ""))
+    target = inner if inner else con_div
+    for img in target.find_all("img"):
+        src = img.get("data-src") or img.get("orig-src") or img.get("src", "")
+        if src:
+            img["src"] = src
+        for attr in ["data-src", "orig-src", "data-width", "data-height"]:
+            if img.has_attr(attr):
+                del img[attr]
+    return str(target.decode_contents()).strip()
+
+
+async def run(limit: int = 50):
+    cfg = get_config()
+    db = Database()
+
+    try:
+        articles = await db.get_articles_without_content(limit=limit)
+        logger.info(f"待采集正文: {len(articles)} 篇")
+
+        if not articles:
+            logger.info("所有文章已有正文，无需采集")
+            return
+
+        success_count = 0
+        fail_count = 0
+
+        for i, art in enumerate(articles, 1):
+            article_id = art["article_id"]
+            title = art["title"][:30]
+            logger.info(f"[{i}/{len(articles)}] article_id={article_id} {title}...")
+
+            try:
+                html = fetch_article_html(article_id)
+                content = extract_content(html)
+
+                if not content:
+                    logger.warning(f"  正文为空，跳过")
+                    fail_count += 1
+                    continue
+
+                await db.update_article_content(article_id, content)
+                success_count += 1
+                logger.info(f"  正文长度: {len(content)} 字符 ✓")
+
+            except Exception as e:
+                logger.error(f"  采集失败: {e}")
+                fail_count += 1
+
+            delay = random.uniform(1.5, 3.0)
+            time.sleep(delay)
+
+        logger.info(f"✅ 正文采集完成: 成功 {success_count}, 失败 {fail_count}")
+
+        await db.log_crawl(
+            crawl_type="article_detail",
+            target="dongqiudi_article_html",
+            success=True,
+            elapsed=0,
+            record_count=success_count,
+        )
+
+    except Exception as e:
+        logger.error(f"采集失败: {e}", exc_info=True)
+        raise
+    finally:
+        await db.close()
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="懂球帝文章正文采集")
+    parser.add_argument("--limit", type=int, default=50, help="最多采集篇数 (默认50)")
+    args = parser.parse_args()
+
+    logger.info("=" * 60)
+    logger.info("懂球帝文章正文采集")
+    logger.info(f"最多采集 {args.limit} 篇")
+    logger.info("=" * 60)
+
+    asyncio.run(run(args.limit))
+
+
+if __name__ == "__main__":
+    main()