Files

194 lines
7.6 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""真题营·考研政治考点真题爬虫 → SQLite
来源: https://zhenti.zalize.com/zhenti/kaodian (2010-2026 按官方考点整理的历年真题, 含答案与解析)
礼貌爬取: 每请求间隔 0.5~0.8s 随机抖动; 失败指数退避重试(1s/2s/4s, 3次); 断点续爬(已入库考点跳过)。
用法: python crawl_zhenti.py (需设置 HTTP_PROXY/HTTPS_PROXY 环境变量走代理)
"""
import os, re, sys, time, random, sqlite3, urllib.parse
import requests
from bs4 import BeautifulSoup
BASE = "https://zhenti.zalize.com"
INDEX = BASE + "/zhenti/kaodian"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9",
}
HERE = os.path.dirname(os.path.abspath(__file__))
DB = os.path.join(HERE, "zhenti_kaodian.db")
BASE_DELAY = 0.5 # 秒, 每次请求前随机 +0~0.3s
def get(url, retries=3):
for i in range(retries):
try:
r = requests.get(url, headers=HEADERS, timeout=30)
if r.status_code == 200:
return r.text
print(f" HTTP {r.status_code} {url}", flush=True)
except Exception as e:
print(f" ERR {type(e).__name__} {url}", flush=True)
time.sleep(2 ** i) # 1s, 2s, 4s 退避
return None
def init_db(con):
con.executescript("""
CREATE TABLE IF NOT EXISTS kaodian(
id INTEGER PRIMARY KEY AUTOINCREMENT,
subject TEXT NOT NULL,
name TEXT NOT NULL UNIQUE,
qcount INTEGER, -- 站点标注的真题数
url TEXT,
crawled_count INTEGER DEFAULT 0,
crawled_at TEXT
);
CREATE TABLE IF NOT EXISTS zhenti(
id INTEGER PRIMARY KEY AUTOINCREMENT,
kaodian_id INTEGER NOT NULL REFERENCES kaodian(id),
year INTEGER,
qno INTEGER,
qtype TEXT, -- 单选 / 多选
stem TEXT, -- 题干
options TEXT, -- JSON: [{"letter","text","correct"}]
answer TEXT, -- 如 ABD
analysis TEXT, -- 解析全文
note TEXT, -- 解析内的争议/备注(如"各机构答案存在分歧")
url TEXT,
UNIQUE(kaodian_id, year, qno)
);
CREATE INDEX IF NOT EXISTS idx_zhenti_kp ON zhenti(kaodian_id);
""")
con.commit()
def parse_index(html):
"""索引页: 提取 科目→考点名→题量→详情URL"""
soup = BeautifulSoup(html, "html.parser")
items = []
for h2 in soup.find_all("h2", id=re.compile(r"^s\d")):
subject = h2.contents[0].strip() if h2.contents else ""
div = h2.find_next_sibling("div")
if not div:
continue
for a in div.find_all("a", href=re.compile(r"^/zhenti/kaodian/")):
label = a.get_text(" ", strip=True)
m = re.search(r"(\d+)\s*$", label)
qc = int(m.group(1)) if m else 0
name = re.sub(r"\s*\d+\s*$", "", label).strip()
items.append({"subject": subject, "name": name, "qcount": qc,
"url": BASE + a["href"]})
return items
def parse_kp_page(html):
"""考点详情页: 逐 article 提取 年份/题号/题型/题干/选项/答案/解析"""
soup = BeautifulSoup(html, "html.parser")
out = []
for art in soup.find_all("article"):
head = art.find("p", class_="font-num")
if not head:
continue
ah = head.find("a", href=re.compile(r"^/zhenti/\d{4}"))
if not ah:
continue
mm = re.match(r"/zhenti/(\d{4})(?:#q(\d+)|/(\d+))", ah["href"])
if not mm:
continue
year, qno = int(mm.group(1)), int(mm.group(2) or mm.group(3))
head_text = head.get_text(" ", strip=True)
qtype = "多选" if "多选" in head_text else ("单选" if "单选" in head_text else "")
stem_el = art.find("p", class_=re.compile(r"mt-1\.5"))
stem = stem_el.get_text(" ", strip=True) if stem_el else ""
opts = []
for p in art.find_all("p", class_="stopt"):
t = p.get_text("", strip=True).replace("✓", "").replace("✗", "").strip()
mopt = re.match(r"^([A-D])\s*[..、]\s*(.*)$", t)
if mopt:
opts.append({"letter": mopt.group(1), "text": mopt.group(2).strip(),
"correct": p.get("data-ok") == "1"})
else:
opts.append({"letter": "", "text": t, "correct": p.get("data-ok") == "1"})
ansbox = art.find("div", class_="ansbox")
answer = analysis = note = ""
if ansbox:
b = ansbox.find("b")
if b:
answer = b.get_text(strip=True).replace("答案", "").strip()
full = ansbox.get_text("\n", strip=True)
full = re.sub(r"^答案\s*[A-D]+\s*", "", full).strip()
nm = re.search(r"注:.*?(?=\n|$)", full)
if nm:
note = nm.group(0).strip()
full = full.replace(note, "").strip()
analysis = re.sub(r"\n{2,}", "\n", full).strip()
out.append({"year": year, "qno": qno, "type": qtype, "stem": stem,
"options": opts, "answer": answer, "analysis": analysis,
"note": note, "url": BASE + f"/zhenti/{year}/{qno}"})
return out
def main():
os.makedirs(HERE, exist_ok=True)
con = sqlite3.connect(DB)
init_db(con)
cur = con.cursor()
idx = get(INDEX)
if not idx:
print("索引页抓取失败", flush=True)
return 1
items = parse_index(idx)
print(f"索引解析: {len(items)} 个考点", flush=True)
cur.execute("SELECT name, crawled_count FROM kaodian")
done = {r[0]: r[1] for r in cur.fetchall()}
for i, it in enumerate(items, 1):
if it["name"] in done and done[it["name"]] > 0:
print(f"[{i}/{len(items)}] 跳过(已爬{done[it['name']]}题): {it['name']}", flush=True)
continue
time.sleep(BASE_DELAY + random.random() * 0.3) # 礼貌限速
html = get(it["url"])
if not html:
print(f"[{i}/{len(items)}] 失败: {it['name']}", flush=True)
continue
qs = parse_kp_page(html)
now = time.strftime("%Y-%m-%d %H:%M:%S")
cur.execute("SELECT id FROM kaodian WHERE name=?", (it["name"],))
row = cur.fetchone()
if row:
kid = row[0]
cur.execute("UPDATE kaodian SET qcount=?, url=?, crawled_count=?, crawled_at=? WHERE id=?",
(it["qcount"], it["url"], len(qs), now, kid))
else:
cur.execute("INSERT INTO kaodian(subject,name,qcount,url,crawled_count,crawled_at) VALUES(?,?,?,?,?,?)",
(it["subject"], it["name"], it["qcount"], it["url"], len(qs), now))
kid = cur.lastrowid
for q in qs:
cur.execute("INSERT OR REPLACE INTO zhenti(kaodian_id,year,qno,qtype,stem,options,answer,analysis,note,url) "
"VALUES(?,?,?,?,?,?,?,?,?,?)",
(kid, q["year"], q["qno"], q["type"], q["stem"],
str(q["options"]), q["answer"], q["analysis"], q["note"], q["url"]))
con.commit()
tag = "✓" if len(qs) == it["qcount"] else f"⚠({len(qs)}/{it['qcount']})"
print(f"[{i}/{len(items)}] {tag} {it['subject']} | {it['name']}", flush=True)
cur.execute("SELECT COUNT(*) FROM zhenti")
n = cur.fetchone()[0]
cur.execute("SELECT COUNT(*) FROM kaodian WHERE crawled_count>0")
k = cur.fetchone()[0]
print(f"完成: 入库 {k} 个考点, zhenti 表共 {n} 题", flush=True)
con.close()
return 0
if __name__ == "__main__":
sys.exit(main())