130 lines
4.8 KiB
Python
130 lines
4.8 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""真题营·考研政治历年分析题爬虫 → SQLite(表 fenxiti)
|
||
来源: https://zhenti.zalize.com/zhenti/fenxiti (2010-2026 分析题 85 道,材料为概述、设问为真题原文、参考答案要点免费预览1/4)
|
||
礼貌爬取: 0.5~0.8s 随机间隔; 指数退避重试; 断点续爬。
|
||
用法: python crawl_fenxiti.py
|
||
"""
|
||
import os, re, time, random, sqlite3, sys
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
|
||
BASE = "https://zhenti.zalize.com"
|
||
INDEX = BASE + "/zhenti/fenxiti"
|
||
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36"}
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
DB = os.path.join(HERE, "zhenti_kaodian.db")
|
||
|
||
|
||
def get(url, retries=3):
|
||
for i in range(retries):
|
||
try:
|
||
r = requests.get(url, headers=HEADERS, timeout=30)
|
||
if r.status_code == 200:
|
||
return r.text
|
||
print(f" HTTP {r.status_code} {url}", flush=True)
|
||
except Exception as e:
|
||
print(f" ERR {type(e).__name__} {url}", flush=True)
|
||
time.sleep(2 ** i)
|
||
return None
|
||
|
||
|
||
def parse_index(html):
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
urls = []
|
||
for a in soup.find_all("a", href=re.compile(r"^/zhenti/fenxiti/\d{4}-\d{2}$")):
|
||
urls.append(BASE + a["href"])
|
||
# 去重保序
|
||
seen = set()
|
||
out = []
|
||
for u in urls:
|
||
if u not in seen:
|
||
seen.add(u)
|
||
out.append(u)
|
||
return out
|
||
|
||
|
||
def parse_page(html):
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
h1 = soup.find("h1")
|
||
m = re.search(r"(\d{4}) 年考研政治分析题第 (\d+) 题", h1.get_text(strip=True) if h1 else "")
|
||
if not m:
|
||
return None
|
||
year, qno = int(m.group(1)), int(m.group(2))
|
||
# 科目与考点: p 文本形如 "马原·哲学 · <a>主观能动性与客观规律性</a> · 材料为原创概述..."
|
||
subj = kp = ""
|
||
p = soup.find("p", class_=re.compile(r"^mt-2 text-sm text-slate-600"))
|
||
if p:
|
||
txt = p.get_text(" ", strip=True)
|
||
subj = txt.split(" · ")[0].strip() if " · " in txt else ""
|
||
kp_a = p.find("a")
|
||
if kp_a:
|
||
kp = kp_a.get_text(strip=True)
|
||
article = soup.find("article")
|
||
stem = article.get_text("\n", strip=True) if article else ""
|
||
ans = ""
|
||
box = soup.find("div", class_=re.compile(r"border-brand-200"))
|
||
if box:
|
||
pts = box.find_all("p", class_="mt-2")
|
||
parts = []
|
||
for pt in pts:
|
||
t = pt.get_text(" ", strip=True)
|
||
if t and "全部要点已整理好" not in t and "注册后可免费" not in t:
|
||
parts.append(t)
|
||
ans = "\n".join(parts)
|
||
return {"year": year, "qno": qno, "subject": subj, "kaodian": kp,
|
||
"stem": stem, "answer_preview": ans, "url": f"{BASE}/zhenti/fenxiti/{year}-{qno}"}
|
||
|
||
|
||
def main():
|
||
con = sqlite3.connect(DB)
|
||
con.executescript("""
|
||
CREATE TABLE IF NOT EXISTS fenxiti(
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
year INTEGER, qno INTEGER,
|
||
subject TEXT, kaodian TEXT,
|
||
stem TEXT, answer_preview TEXT, url TEXT,
|
||
UNIQUE(year, qno)
|
||
);
|
||
""")
|
||
con.commit()
|
||
cur = con.cursor()
|
||
idx = get(INDEX)
|
||
if not idx:
|
||
print("索引页失败", flush=True)
|
||
return 1
|
||
urls = parse_index(idx)
|
||
print(f"分析题索引: {len(urls)} 道", flush=True)
|
||
cur.execute("SELECT year, qno, subject, kaodian FROM fenxiti")
|
||
done = {(r[0], r[1]): (r[2], r[3]) for r in cur.fetchall()}
|
||
bad_kp = {"哲学", "毛中特", "史纲", "思修法基", "形势与政策", "马原"}
|
||
for i, u in enumerate(urls, 1):
|
||
m = re.search(r"/(\d{4})-(\d{2})$", u)
|
||
key = (int(m.group(1)), int(m.group(2)))
|
||
if key in done and done[key][0] and done[key][1] and done[key][1] not in bad_kp:
|
||
print(f"[{i}/{len(urls)}] 跳过: {key}", flush=True)
|
||
continue
|
||
time.sleep(0.5 + random.random() * 0.3)
|
||
html = get(u)
|
||
if not html:
|
||
print(f"[{i}/{len(urls)}] 失败: {u}", flush=True)
|
||
continue
|
||
q = parse_page(html)
|
||
if not q:
|
||
print(f"[{i}/{len(urls)}] 解析失败: {u}", flush=True)
|
||
continue
|
||
cur.execute("INSERT OR REPLACE INTO fenxiti(year,qno,subject,kaodian,stem,answer_preview,url) "
|
||
"VALUES(?,?,?,?,?,?,?)",
|
||
(q["year"], q["qno"], q["subject"], q["kaodian"], q["stem"], q["answer_preview"], q["url"]))
|
||
con.commit()
|
||
print(f"[{i}/{len(urls)}] ✓ {q['year']}-{q['qno']} {q['subject']} | {q['kaodian']}", flush=True)
|
||
cur.execute("SELECT COUNT(*) FROM fenxiti")
|
||
print(f"完成: fenxiti 表共 {cur.fetchone()[0]} 道", flush=True)
|
||
con.close()
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|