# -*- coding: utf-8 -*- """真题营·考研政治考点真题爬虫 → SQLite 来源: https://zhenti.zalize.com/zhenti/kaodian (2010-2026 按官方考点整理的历年真题, 含答案与解析) 礼貌爬取: 每请求间隔 0.5~0.8s 随机抖动; 失败指数退避重试(1s/2s/4s, 3次); 断点续爬(已入库考点跳过)。 用法: python crawl_zhenti.py (需设置 HTTP_PROXY/HTTPS_PROXY 环境变量走代理) """ import os, re, sys, time, random, sqlite3, urllib.parse import requests from bs4 import BeautifulSoup BASE = "https://zhenti.zalize.com" INDEX = BASE + "/zhenti/kaodian" HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0 Safari/537.36", "Accept-Language": "zh-CN,zh;q=0.9", } HERE = os.path.dirname(os.path.abspath(__file__)) DB = os.path.join(HERE, "zhenti_kaodian.db") BASE_DELAY = 0.5 # 秒, 每次请求前随机 +0~0.3s def get(url, retries=3): for i in range(retries): try: r = requests.get(url, headers=HEADERS, timeout=30) if r.status_code == 200: return r.text print(f" HTTP {r.status_code} {url}", flush=True) except Exception as e: print(f" ERR {type(e).__name__} {url}", flush=True) time.sleep(2 ** i) # 1s, 2s, 4s 退避 return None def init_db(con): con.executescript(""" CREATE TABLE IF NOT EXISTS kaodian( id INTEGER PRIMARY KEY AUTOINCREMENT, subject TEXT NOT NULL, name TEXT NOT NULL UNIQUE, qcount INTEGER, -- 站点标注的真题数 url TEXT, crawled_count INTEGER DEFAULT 0, crawled_at TEXT ); CREATE TABLE IF NOT EXISTS zhenti( id INTEGER PRIMARY KEY AUTOINCREMENT, kaodian_id INTEGER NOT NULL REFERENCES kaodian(id), year INTEGER, qno INTEGER, qtype TEXT, -- 单选 / 多选 stem TEXT, -- 题干 options TEXT, -- JSON: [{"letter","text","correct"}] answer TEXT, -- 如 ABD analysis TEXT, -- 解析全文 note TEXT, -- 解析内的争议/备注(如"各机构答案存在分歧") url TEXT, UNIQUE(kaodian_id, year, qno) ); CREATE INDEX IF NOT EXISTS idx_zhenti_kp ON zhenti(kaodian_id); """) con.commit() def parse_index(html): """索引页: 提取 科目→考点名→题量→详情URL""" soup = BeautifulSoup(html, "html.parser") items = [] for h2 in soup.find_all("h2", id=re.compile(r"^s\d")): subject = h2.contents[0].strip() if h2.contents else "" div = h2.find_next_sibling("div") if not div: continue for a in div.find_all("a", href=re.compile(r"^/zhenti/kaodian/")): label = a.get_text(" ", strip=True) m = re.search(r"(\d+)\s*$", label) qc = int(m.group(1)) if m else 0 name = re.sub(r"\s*\d+\s*$", "", label).strip() items.append({"subject": subject, "name": name, "qcount": qc, "url": BASE + a["href"]}) return items def parse_kp_page(html): """考点详情页: 逐 article 提取 年份/题号/题型/题干/选项/答案/解析""" soup = BeautifulSoup(html, "html.parser") out = [] for art in soup.find_all("article"): head = art.find("p", class_="font-num") if not head: continue ah = head.find("a", href=re.compile(r"^/zhenti/\d{4}")) if not ah: continue mm = re.match(r"/zhenti/(\d{4})(?:#q(\d+)|/(\d+))", ah["href"]) if not mm: continue year, qno = int(mm.group(1)), int(mm.group(2) or mm.group(3)) head_text = head.get_text(" ", strip=True) qtype = "多选" if "多选" in head_text else ("单选" if "单选" in head_text else "") stem_el = art.find("p", class_=re.compile(r"mt-1\.5")) stem = stem_el.get_text(" ", strip=True) if stem_el else "" opts = [] for p in art.find_all("p", class_="stopt"): t = p.get_text("", strip=True).replace("✓", "").replace("✗", "").strip() mopt = re.match(r"^([A-D])\s*[..、]\s*(.*)$", t) if mopt: opts.append({"letter": mopt.group(1), "text": mopt.group(2).strip(), "correct": p.get("data-ok") == "1"}) else: opts.append({"letter": "", "text": t, "correct": p.get("data-ok") == "1"}) ansbox = art.find("div", class_="ansbox") answer = analysis = note = "" if ansbox: b = ansbox.find("b") if b: answer = b.get_text(strip=True).replace("答案", "").strip() full = ansbox.get_text("\n", strip=True) full = re.sub(r"^答案\s*[A-D]+\s*", "", full).strip() nm = re.search(r"注:.*?(?=\n|$)", full) if nm: note = nm.group(0).strip() full = full.replace(note, "").strip() analysis = re.sub(r"\n{2,}", "\n", full).strip() out.append({"year": year, "qno": qno, "type": qtype, "stem": stem, "options": opts, "answer": answer, "analysis": analysis, "note": note, "url": BASE + f"/zhenti/{year}/{qno}"}) return out def main(): os.makedirs(HERE, exist_ok=True) con = sqlite3.connect(DB) init_db(con) cur = con.cursor() idx = get(INDEX) if not idx: print("索引页抓取失败", flush=True) return 1 items = parse_index(idx) print(f"索引解析: {len(items)} 个考点", flush=True) cur.execute("SELECT name, crawled_count FROM kaodian") done = {r[0]: r[1] for r in cur.fetchall()} for i, it in enumerate(items, 1): if it["name"] in done and done[it["name"]] > 0: print(f"[{i}/{len(items)}] 跳过(已爬{done[it['name']]}题): {it['name']}", flush=True) continue time.sleep(BASE_DELAY + random.random() * 0.3) # 礼貌限速 html = get(it["url"]) if not html: print(f"[{i}/{len(items)}] 失败: {it['name']}", flush=True) continue qs = parse_kp_page(html) now = time.strftime("%Y-%m-%d %H:%M:%S") cur.execute("SELECT id FROM kaodian WHERE name=?", (it["name"],)) row = cur.fetchone() if row: kid = row[0] cur.execute("UPDATE kaodian SET qcount=?, url=?, crawled_count=?, crawled_at=? WHERE id=?", (it["qcount"], it["url"], len(qs), now, kid)) else: cur.execute("INSERT INTO kaodian(subject,name,qcount,url,crawled_count,crawled_at) VALUES(?,?,?,?,?,?)", (it["subject"], it["name"], it["qcount"], it["url"], len(qs), now)) kid = cur.lastrowid for q in qs: cur.execute("INSERT OR REPLACE INTO zhenti(kaodian_id,year,qno,qtype,stem,options,answer,analysis,note,url) " "VALUES(?,?,?,?,?,?,?,?,?,?)", (kid, q["year"], q["qno"], q["type"], q["stem"], str(q["options"]), q["answer"], q["analysis"], q["note"], q["url"])) con.commit() tag = "✓" if len(qs) == it["qcount"] else f"⚠({len(qs)}/{it['qcount']})" print(f"[{i}/{len(items)}] {tag} {it['subject']} | {it['name']}", flush=True) cur.execute("SELECT COUNT(*) FROM zhenti") n = cur.fetchone()[0] cur.execute("SELECT COUNT(*) FROM kaodian WHERE crawled_count>0") k = cur.fetchone()[0] print(f"完成: 入库 {k} 个考点, zhenti 表共 {n} 题", flush=True) con.close() return 0 if __name__ == "__main__": sys.exit(main())