真题入库SQLite(544题)+马原9专题内嵌真题精练(134题),修复双链别名分隔符,规范更新

This commit is contained in:
2026-10-26 22:29:07 +08:00
parent b5d451a49d
commit 8f2391b16d
22 changed files with 1448 additions and 54 deletions
+193
View File
@@ -0,0 +1,193 @@
# -*- coding: utf-8 -*-
"""真题营·考研政治考点真题爬虫 → SQLite
来源: https://zhenti.zalize.com/zhenti/kaodian (2010-2026 按官方考点整理的历年真题, 含答案与解析)
礼貌爬取: 每请求间隔 0.5~0.8s 随机抖动; 失败指数退避重试(1s/2s/4s, 3次); 断点续爬(已入库考点跳过)。
用法: python crawl_zhenti.py (需设置 HTTP_PROXY/HTTPS_PROXY 环境变量走代理)
"""
import os, re, sys, time, random, sqlite3, urllib.parse
import requests
from bs4 import BeautifulSoup
BASE = "https://zhenti.zalize.com"
INDEX = BASE + "/zhenti/kaodian"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9",
}
HERE = os.path.dirname(os.path.abspath(__file__))
DB = os.path.join(HERE, "zhenti_kaodian.db")
BASE_DELAY = 0.5 # 秒, 每次请求前随机 +0~0.3s
def get(url, retries=3):
for i in range(retries):
try:
r = requests.get(url, headers=HEADERS, timeout=30)
if r.status_code == 200:
return r.text
print(f" HTTP {r.status_code} {url}", flush=True)
except Exception as e:
print(f" ERR {type(e).__name__} {url}", flush=True)
time.sleep(2 ** i) # 1s, 2s, 4s 退避
return None
def init_db(con):
con.executescript("""
CREATE TABLE IF NOT EXISTS kaodian(
id INTEGER PRIMARY KEY AUTOINCREMENT,
subject TEXT NOT NULL,
name TEXT NOT NULL UNIQUE,
qcount INTEGER, -- 站点标注的真题数
url TEXT,
crawled_count INTEGER DEFAULT 0,
crawled_at TEXT
);
CREATE TABLE IF NOT EXISTS zhenti(
id INTEGER PRIMARY KEY AUTOINCREMENT,
kaodian_id INTEGER NOT NULL REFERENCES kaodian(id),
year INTEGER,
qno INTEGER,
qtype TEXT, -- 单选 / 多选
stem TEXT, -- 题干
options TEXT, -- JSON: [{"letter","text","correct"}]
answer TEXT, -- 如 ABD
analysis TEXT, -- 解析全文
note TEXT, -- 解析内的争议/备注(如"各机构答案存在分歧")
url TEXT,
UNIQUE(kaodian_id, year, qno)
);
CREATE INDEX IF NOT EXISTS idx_zhenti_kp ON zhenti(kaodian_id);
""")
con.commit()
def parse_index(html):
"""索引页: 提取 科目→考点名→题量→详情URL"""
soup = BeautifulSoup(html, "html.parser")
items = []
for h2 in soup.find_all("h2", id=re.compile(r"^s\d")):
subject = h2.contents[0].strip() if h2.contents else ""
div = h2.find_next_sibling("div")
if not div:
continue
for a in div.find_all("a", href=re.compile(r"^/zhenti/kaodian/")):
label = a.get_text(" ", strip=True)
m = re.search(r"(\d+)\s*$", label)
qc = int(m.group(1)) if m else 0
name = re.sub(r"\s*\d+\s*$", "", label).strip()
items.append({"subject": subject, "name": name, "qcount": qc,
"url": BASE + a["href"]})
return items
def parse_kp_page(html):
"""考点详情页: 逐 article 提取 年份/题号/题型/题干/选项/答案/解析"""
soup = BeautifulSoup(html, "html.parser")
out = []
for art in soup.find_all("article"):
head = art.find("p", class_="font-num")
if not head:
continue
ah = head.find("a", href=re.compile(r"^/zhenti/\d{4}"))
if not ah:
continue
mm = re.match(r"/zhenti/(\d{4})(?:#q(\d+)|/(\d+))", ah["href"])
if not mm:
continue
year, qno = int(mm.group(1)), int(mm.group(2) or mm.group(3))
head_text = head.get_text(" ", strip=True)
qtype = "多选" if "多选" in head_text else ("单选" if "单选" in head_text else "")
stem_el = art.find("p", class_=re.compile(r"mt-1\.5"))
stem = stem_el.get_text(" ", strip=True) if stem_el else ""
opts = []
for p in art.find_all("p", class_="stopt"):
t = p.get_text("", strip=True).replace("✓", "").replace("✗", "").strip()
mopt = re.match(r"^([A-D])\s*[..、]\s*(.*)$", t)
if mopt:
opts.append({"letter": mopt.group(1), "text": mopt.group(2).strip(),
"correct": p.get("data-ok") == "1"})
else:
opts.append({"letter": "", "text": t, "correct": p.get("data-ok") == "1"})
ansbox = art.find("div", class_="ansbox")
answer = analysis = note = ""
if ansbox:
b = ansbox.find("b")
if b:
answer = b.get_text(strip=True).replace("答案", "").strip()
full = ansbox.get_text("\n", strip=True)
full = re.sub(r"^答案\s*[A-D]+\s*", "", full).strip()
nm = re.search(r"注:.*?(?=\n|$)", full)
if nm:
note = nm.group(0).strip()
full = full.replace(note, "").strip()
analysis = re.sub(r"\n{2,}", "\n", full).strip()
out.append({"year": year, "qno": qno, "type": qtype, "stem": stem,
"options": opts, "answer": answer, "analysis": analysis,
"note": note, "url": BASE + f"/zhenti/{year}/{qno}"})
return out
def main():
os.makedirs(HERE, exist_ok=True)
con = sqlite3.connect(DB)
init_db(con)
cur = con.cursor()
idx = get(INDEX)
if not idx:
print("索引页抓取失败", flush=True)
return 1
items = parse_index(idx)
print(f"索引解析: {len(items)} 个考点", flush=True)
cur.execute("SELECT name, crawled_count FROM kaodian")
done = {r[0]: r[1] for r in cur.fetchall()}
for i, it in enumerate(items, 1):
if it["name"] in done and done[it["name"]] > 0:
print(f"[{i}/{len(items)}] 跳过(已爬{done[it['name']]}题): {it['name']}", flush=True)
continue
time.sleep(BASE_DELAY + random.random() * 0.3) # 礼貌限速
html = get(it["url"])
if not html:
print(f"[{i}/{len(items)}] 失败: {it['name']}", flush=True)
continue
qs = parse_kp_page(html)
now = time.strftime("%Y-%m-%d %H:%M:%S")
cur.execute("SELECT id FROM kaodian WHERE name=?", (it["name"],))
row = cur.fetchone()
if row:
kid = row[0]
cur.execute("UPDATE kaodian SET qcount=?, url=?, crawled_count=?, crawled_at=? WHERE id=?",
(it["qcount"], it["url"], len(qs), now, kid))
else:
cur.execute("INSERT INTO kaodian(subject,name,qcount,url,crawled_count,crawled_at) VALUES(?,?,?,?,?,?)",
(it["subject"], it["name"], it["qcount"], it["url"], len(qs), now))
kid = cur.lastrowid
for q in qs:
cur.execute("INSERT OR REPLACE INTO zhenti(kaodian_id,year,qno,qtype,stem,options,answer,analysis,note,url) "
"VALUES(?,?,?,?,?,?,?,?,?,?)",
(kid, q["year"], q["qno"], q["type"], q["stem"],
str(q["options"]), q["answer"], q["analysis"], q["note"], q["url"]))
con.commit()
tag = "✓" if len(qs) == it["qcount"] else f"⚠({len(qs)}/{it['qcount']})"
print(f"[{i}/{len(items)}] {tag} {it['subject']} | {it['name']}", flush=True)
cur.execute("SELECT COUNT(*) FROM zhenti")
n = cur.fetchone()[0]
cur.execute("SELECT COUNT(*) FROM kaodian WHERE crawled_count>0")
k = cur.fetchone()[0]
print(f"完成: 入库 {k} 个考点, zhenti 表共 {n} 题", flush=True)
con.close()
return 0
if __name__ == "__main__":
sys.exit(main())