136 lines
4.1 KiB
Python
136 lines
4.1 KiB
Python
"""
|
|
懂球帝文章正文采集脚本
|
|
- 从数据库读取已入库但无正文的文章
|
|
- 请求文章详情页HTML，提取 div.con 正文内容
|
|
- 将正文HTML写入 la_article.content
|
|
用法:
|
|
python scripts/crawl_article_detail.py # 采集最多50篇
|
|
python scripts/crawl_article_detail.py --limit 200 # 采集最多200篇
|
|
"""
|
|
import asyncio
|
|
import random
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from bs4 import BeautifulSoup
|
|
from curl_cffi import requests as curl_requests
|
|
from loguru import logger
|
|
|
|
from src.core.config import get_config
|
|
from src.storage.database import Database
|
|
|
|
DETAIL_URL = "https://www.dongqiudi.com/articles/{article_id}"
|
|
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
"Referer": "https://www.dongqiudi.com/",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
}
|
|
|
|
|
|
def fetch_article_html(article_id: int) -> str:
|
|
url = DETAIL_URL.format(article_id=article_id)
|
|
resp = curl_requests.get(
|
|
url, headers=HEADERS,
|
|
impersonate="chrome", timeout=20,
|
|
)
|
|
resp.raise_for_status()
|
|
return resp.text
|
|
|
|
|
|
def extract_content(html: str) -> str:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
con_div = soup.select_one("div.con")
|
|
if not con_div:
|
|
return ""
|
|
inner = con_div.find("div", style=lambda v: v and "display:none" in v.replace(" ", ""))
|
|
target = inner if inner else con_div
|
|
for img in target.find_all("img"):
|
|
src = img.get("data-src") or img.get("orig-src") or img.get("src", "")
|
|
if src:
|
|
img["src"] = src
|
|
for attr in ["data-src", "orig-src", "data-width", "data-height"]:
|
|
if img.has_attr(attr):
|
|
del img[attr]
|
|
return str(target.decode_contents()).strip()
|
|
|
|
|
|
async def run(limit: int = 50):
|
|
cfg = get_config()
|
|
db = Database()
|
|
|
|
try:
|
|
articles = await db.get_articles_without_content(limit=limit)
|
|
logger.info(f"待采集正文: {len(articles)} 篇")
|
|
|
|
if not articles:
|
|
logger.info("所有文章已有正文，无需采集")
|
|
return
|
|
|
|
success_count = 0
|
|
fail_count = 0
|
|
|
|
for i, art in enumerate(articles, 1):
|
|
article_id = art["article_id"]
|
|
title = art["title"][:30]
|
|
logger.info(f"[{i}/{len(articles)}] article_id={article_id} {title}...")
|
|
|
|
try:
|
|
html = fetch_article_html(article_id)
|
|
content = extract_content(html)
|
|
|
|
if not content:
|
|
logger.warning(f" 正文为空，跳过")
|
|
fail_count += 1
|
|
continue
|
|
|
|
await db.update_article_content(article_id, content)
|
|
success_count += 1
|
|
logger.info(f" 正文长度: {len(content)} 字符 ✓")
|
|
|
|
except Exception as e:
|
|
logger.error(f" 采集失败: {e}")
|
|
fail_count += 1
|
|
|
|
delay = random.uniform(1.5, 3.0)
|
|
time.sleep(delay)
|
|
|
|
logger.info(f"✅ 正文采集完成: 成功 {success_count}, 失败 {fail_count}")
|
|
|
|
await db.log_crawl(
|
|
crawl_type="article_detail",
|
|
target="dongqiudi_article_html",
|
|
success=True,
|
|
elapsed=0,
|
|
record_count=success_count,
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"采集失败: {e}", exc_info=True)
|
|
raise
|
|
finally:
|
|
await db.close()
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description="懂球帝文章正文采集")
|
|
parser.add_argument("--limit", type=int, default=50, help="最多采集篇数 (默认50)")
|
|
args = parser.parse_args()
|
|
|
|
logger.info("=" * 60)
|
|
logger.info("懂球帝文章正文采集")
|
|
logger.info(f"最多采集 {args.limit} 篇")
|
|
logger.info("=" * 60)
|
|
|
|
asyncio.run(run(args.limit))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|