130 lines
4.8 KiB
Python
130 lines
4.8 KiB
Python
# -*- coding: utf-8 -*-
|
||||
|
|
"""真题营·考研政治历年分析题爬虫 → SQLite(表 fenxiti)
|
|||
|
|
来源: https://zhenti.zalize.com/zhenti/fenxiti (2010-2026 分析题 85 道,材料为概述、设问为真题原文、参考答案要点免费预览1/4)
|
|||
|
|
礼貌爬取: 0.5~0.8s 随机间隔; 指数退避重试; 断点续爬。
|
|||
|
|
用法: python crawl_fenxiti.py
|
|||
|
|
"""
|
|||
|
|
import os, re, time, random, sqlite3, sys
|
|||
|
|
|
|||
|
|
import requests
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
|
|||
|
|
BASE = "https://zhenti.zalize.com"
|
|||
|
|
INDEX = BASE + "/zhenti/fenxiti"
|
|||
|
|
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|||
|
|
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36"}
|
|||
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|||
|
|
DB = os.path.join(HERE, "zhenti_kaodian.db")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get(url, retries=3):
|
|||
|
|
for i in range(retries):
|
|||
|
|
try:
|
|||
|
|
r = requests.get(url, headers=HEADERS, timeout=30)
|
|||
|
|
if r.status_code == 200:
|
|||
|
|
return r.text
|
|||
|
|
print(f" HTTP {r.status_code} {url}", flush=True)
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f" ERR {type(e).__name__} {url}", flush=True)
|
|||
|
|
time.sleep(2 ** i)
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_index(html):
|
|||
|
|
soup = BeautifulSoup(html, "html.parser")
|
|||
|
|
urls = []
|
|||
|
|
for a in soup.find_all("a", href=re.compile(r"^/zhenti/fenxiti/\d{4}-\d{2}$")):
|
|||
|
|
urls.append(BASE + a["href"])
|
|||
|
|
# 去重保序
|
|||
|
|
seen = set()
|
|||
|
|
out = []
|
|||
|
|
for u in urls:
|
|||
|
|
if u not in seen:
|
|||
|
|
seen.add(u)
|
|||
|
|
out.append(u)
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_page(html):
|
|||
|
|
soup = BeautifulSoup(html, "html.parser")
|
|||
|
|
h1 = soup.find("h1")
|
|||
|
|
m = re.search(r"(\d{4}) 年考研政治分析题第 (\d+) 题", h1.get_text(strip=True) if h1 else "")
|
|||
|
|
if not m:
|
|||
|
|
return None
|
|||
|
|
year, qno = int(m.group(1)), int(m.group(2))
|
|||
|
|
# 科目与考点: p 文本形如 "马原·哲学 · <a>主观能动性与客观规律性</a> · 材料为原创概述..."
|
|||
|
|
subj = kp = ""
|
|||
|
|
p = soup.find("p", class_=re.compile(r"^mt-2 text-sm text-slate-600"))
|
|||
|
|
if p:
|
|||
|
|
txt = p.get_text(" ", strip=True)
|
|||
|
|
subj = txt.split(" · ")[0].strip() if " · " in txt else ""
|
|||
|
|
kp_a = p.find("a")
|
|||
|
|
if kp_a:
|
|||
|
|
kp = kp_a.get_text(strip=True)
|
|||
|
|
article = soup.find("article")
|
|||
|
|
stem = article.get_text("\n", strip=True) if article else ""
|
|||
|
|
ans = ""
|
|||
|
|
box = soup.find("div", class_=re.compile(r"border-brand-200"))
|
|||
|
|
if box:
|
|||
|
|
pts = box.find_all("p", class_="mt-2")
|
|||
|
|
parts = []
|
|||
|
|
for pt in pts:
|
|||
|
|
t = pt.get_text(" ", strip=True)
|
|||
|
|
if t and "全部要点已整理好" not in t and "注册后可免费" not in t:
|
|||
|
|
parts.append(t)
|
|||
|
|
ans = "\n".join(parts)
|
|||
|
|
return {"year": year, "qno": qno, "subject": subj, "kaodian": kp,
|
|||
|
|
"stem": stem, "answer_preview": ans, "url": f"{BASE}/zhenti/fenxiti/{year}-{qno}"}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
con = sqlite3.connect(DB)
|
|||
|
|
con.executescript("""
|
|||
|
|
CREATE TABLE IF NOT EXISTS fenxiti(
|
|||
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|||
|
|
year INTEGER, qno INTEGER,
|
|||
|
|
subject TEXT, kaodian TEXT,
|
|||
|
|
stem TEXT, answer_preview TEXT, url TEXT,
|
|||
|
|
UNIQUE(year, qno)
|
|||
|
|
);
|
|||
|
|
""")
|
|||
|
|
con.commit()
|
|||
|
|
cur = con.cursor()
|
|||
|
|
idx = get(INDEX)
|
|||
|
|
if not idx:
|
|||
|
|
print("索引页失败", flush=True)
|
|||
|
|
return 1
|
|||
|
|
urls = parse_index(idx)
|
|||
|
|
print(f"分析题索引: {len(urls)} 道", flush=True)
|
|||
|
|
cur.execute("SELECT year, qno, subject, kaodian FROM fenxiti")
|
|||
|
|
done = {(r[0], r[1]): (r[2], r[3]) for r in cur.fetchall()}
|
|||
|
|
bad_kp = {"哲学", "毛中特", "史纲", "思修法基", "形势与政策", "马原"}
|
|||
|
|
for i, u in enumerate(urls, 1):
|
|||
|
|
m = re.search(r"/(\d{4})-(\d{2})$", u)
|
|||
|
|
key = (int(m.group(1)), int(m.group(2)))
|
|||
|
|
if key in done and done[key][0] and done[key][1] and done[key][1] not in bad_kp:
|
|||
|
|
print(f"[{i}/{len(urls)}] 跳过: {key}", flush=True)
|
|||
|
|
continue
|
|||
|
|
time.sleep(0.5 + random.random() * 0.3)
|
|||
|
|
html = get(u)
|
|||
|
|
if not html:
|
|||
|
|
print(f"[{i}/{len(urls)}] 失败: {u}", flush=True)
|
|||
|
|
continue
|
|||
|
|
q = parse_page(html)
|
|||
|
|
if not q:
|
|||
|
|
print(f"[{i}/{len(urls)}] 解析失败: {u}", flush=True)
|
|||
|
|
continue
|
|||
|
|
cur.execute("INSERT OR REPLACE INTO fenxiti(year,qno,subject,kaodian,stem,answer_preview,url) "
|
|||
|
|
"VALUES(?,?,?,?,?,?,?)",
|
|||
|
|
(q["year"], q["qno"], q["subject"], q["kaodian"], q["stem"], q["answer_preview"], q["url"]))
|
|||
|
|
con.commit()
|
|||
|
|
print(f"[{i}/{len(urls)}] ✓ {q['year']}-{q['qno']} {q['subject']} | {q['kaodian']}", flush=True)
|
|||
|
|
cur.execute("SELECT COUNT(*) FROM fenxiti")
|
|||
|
|
print(f"完成: fenxiti 表共 {cur.fetchone()[0]} 道", flush=True)
|
|||
|
|
con.close()
|
|||
|
|
return 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
sys.exit(main())
|