# -*- coding: utf-8 -*- """真题营·考研政治历年分析题爬虫 → SQLite(表 fenxiti) 来源: https://zhenti.zalize.com/zhenti/fenxiti (2010-2026 分析题 85 道,材料为概述、设问为真题原文、参考答案要点免费预览1/4) 礼貌爬取: 0.5~0.8s 随机间隔; 指数退避重试; 断点续爬。 用法: python crawl_fenxiti.py """ import os, re, time, random, sqlite3, sys import requests from bs4 import BeautifulSoup BASE = "https://zhenti.zalize.com" INDEX = BASE + "/zhenti/fenxiti" HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0 Safari/537.36"} HERE = os.path.dirname(os.path.abspath(__file__)) DB = os.path.join(HERE, "zhenti_kaodian.db") def get(url, retries=3): for i in range(retries): try: r = requests.get(url, headers=HEADERS, timeout=30) if r.status_code == 200: return r.text print(f" HTTP {r.status_code} {url}", flush=True) except Exception as e: print(f" ERR {type(e).__name__} {url}", flush=True) time.sleep(2 ** i) return None def parse_index(html): soup = BeautifulSoup(html, "html.parser") urls = [] for a in soup.find_all("a", href=re.compile(r"^/zhenti/fenxiti/\d{4}-\d{2}$")): urls.append(BASE + a["href"]) # 去重保序 seen = set() out = [] for u in urls: if u not in seen: seen.add(u) out.append(u) return out def parse_page(html): soup = BeautifulSoup(html, "html.parser") h1 = soup.find("h1") m = re.search(r"(\d{4}) 年考研政治分析题第 (\d+) 题", h1.get_text(strip=True) if h1 else "") if not m: return None year, qno = int(m.group(1)), int(m.group(2)) # 科目与考点: p 文本形如 "马原·哲学 · 主观能动性与客观规律性 · 材料为原创概述..." subj = kp = "" p = soup.find("p", class_=re.compile(r"^mt-2 text-sm text-slate-600")) if p: txt = p.get_text(" ", strip=True) subj = txt.split(" · ")[0].strip() if " · " in txt else "" kp_a = p.find("a") if kp_a: kp = kp_a.get_text(strip=True) article = soup.find("article") stem = article.get_text("\n", strip=True) if article else "" ans = "" box = soup.find("div", class_=re.compile(r"border-brand-200")) if box: pts = box.find_all("p", class_="mt-2") parts = [] for pt in pts: t = pt.get_text(" ", strip=True) if t and "全部要点已整理好" not in t and "注册后可免费" not in t: parts.append(t) ans = "\n".join(parts) return {"year": year, "qno": qno, "subject": subj, "kaodian": kp, "stem": stem, "answer_preview": ans, "url": f"{BASE}/zhenti/fenxiti/{year}-{qno}"} def main(): con = sqlite3.connect(DB) con.executescript(""" CREATE TABLE IF NOT EXISTS fenxiti( id INTEGER PRIMARY KEY AUTOINCREMENT, year INTEGER, qno INTEGER, subject TEXT, kaodian TEXT, stem TEXT, answer_preview TEXT, url TEXT, UNIQUE(year, qno) ); """) con.commit() cur = con.cursor() idx = get(INDEX) if not idx: print("索引页失败", flush=True) return 1 urls = parse_index(idx) print(f"分析题索引: {len(urls)} 道", flush=True) cur.execute("SELECT year, qno, subject, kaodian FROM fenxiti") done = {(r[0], r[1]): (r[2], r[3]) for r in cur.fetchall()} bad_kp = {"哲学", "毛中特", "史纲", "思修法基", "形势与政策", "马原"} for i, u in enumerate(urls, 1): m = re.search(r"/(\d{4})-(\d{2})$", u) key = (int(m.group(1)), int(m.group(2))) if key in done and done[key][0] and done[key][1] and done[key][1] not in bad_kp: print(f"[{i}/{len(urls)}] 跳过: {key}", flush=True) continue time.sleep(0.5 + random.random() * 0.3) html = get(u) if not html: print(f"[{i}/{len(urls)}] 失败: {u}", flush=True) continue q = parse_page(html) if not q: print(f"[{i}/{len(urls)}] 解析失败: {u}", flush=True) continue cur.execute("INSERT OR REPLACE INTO fenxiti(year,qno,subject,kaodian,stem,answer_preview,url) " "VALUES(?,?,?,?,?,?,?)", (q["year"], q["qno"], q["subject"], q["kaodian"], q["stem"], q["answer_preview"], q["url"])) con.commit() print(f"[{i}/{len(urls)}] ✓ {q['year']}-{q['qno']} {q['subject']} | {q['kaodian']}", flush=True) cur.execute("SELECT COUNT(*) FROM fenxiti") print(f"完成: fenxiti 表共 {cur.fetchone()[0]} 道", flush=True) con.close() return 0 if __name__ == "__main__": sys.exit(main())