194 lines
7.6 KiB
Python
194 lines
7.6 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""真题营·考研政治考点真题爬虫 → SQLite
|
||
来源: https://zhenti.zalize.com/zhenti/kaodian (2010-2026 按官方考点整理的历年真题, 含答案与解析)
|
||
礼貌爬取: 每请求间隔 0.5~0.8s 随机抖动; 失败指数退避重试(1s/2s/4s, 3次); 断点续爬(已入库考点跳过)。
|
||
用法: python crawl_zhenti.py (需设置 HTTP_PROXY/HTTPS_PROXY 环境变量走代理)
|
||
"""
|
||
import os, re, sys, time, random, sqlite3, urllib.parse
|
||
|
||
import requests
|
||
from bs4 import BeautifulSoup
|
||
|
||
BASE = "https://zhenti.zalize.com"
|
||
INDEX = BASE + "/zhenti/kaodian"
|
||
HEADERS = {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36",
|
||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||
}
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
DB = os.path.join(HERE, "zhenti_kaodian.db")
|
||
BASE_DELAY = 0.5 # 秒, 每次请求前随机 +0~0.3s
|
||
|
||
|
||
def get(url, retries=3):
|
||
for i in range(retries):
|
||
try:
|
||
r = requests.get(url, headers=HEADERS, timeout=30)
|
||
if r.status_code == 200:
|
||
return r.text
|
||
print(f" HTTP {r.status_code} {url}", flush=True)
|
||
except Exception as e:
|
||
print(f" ERR {type(e).__name__} {url}", flush=True)
|
||
time.sleep(2 ** i) # 1s, 2s, 4s 退避
|
||
return None
|
||
|
||
|
||
def init_db(con):
|
||
con.executescript("""
|
||
CREATE TABLE IF NOT EXISTS kaodian(
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
subject TEXT NOT NULL,
|
||
name TEXT NOT NULL UNIQUE,
|
||
qcount INTEGER, -- 站点标注的真题数
|
||
url TEXT,
|
||
crawled_count INTEGER DEFAULT 0,
|
||
crawled_at TEXT
|
||
);
|
||
CREATE TABLE IF NOT EXISTS zhenti(
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
kaodian_id INTEGER NOT NULL REFERENCES kaodian(id),
|
||
year INTEGER,
|
||
qno INTEGER,
|
||
qtype TEXT, -- 单选 / 多选
|
||
stem TEXT, -- 题干
|
||
options TEXT, -- JSON: [{"letter","text","correct"}]
|
||
answer TEXT, -- 如 ABD
|
||
analysis TEXT, -- 解析全文
|
||
note TEXT, -- 解析内的争议/备注(如"各机构答案存在分歧")
|
||
url TEXT,
|
||
UNIQUE(kaodian_id, year, qno)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_zhenti_kp ON zhenti(kaodian_id);
|
||
""")
|
||
con.commit()
|
||
|
||
|
||
def parse_index(html):
|
||
"""索引页: 提取 科目→考点名→题量→详情URL"""
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
items = []
|
||
for h2 in soup.find_all("h2", id=re.compile(r"^s\d")):
|
||
subject = h2.contents[0].strip() if h2.contents else ""
|
||
div = h2.find_next_sibling("div")
|
||
if not div:
|
||
continue
|
||
for a in div.find_all("a", href=re.compile(r"^/zhenti/kaodian/")):
|
||
label = a.get_text(" ", strip=True)
|
||
m = re.search(r"(\d+)\s*$", label)
|
||
qc = int(m.group(1)) if m else 0
|
||
name = re.sub(r"\s*\d+\s*$", "", label).strip()
|
||
items.append({"subject": subject, "name": name, "qcount": qc,
|
||
"url": BASE + a["href"]})
|
||
return items
|
||
|
||
|
||
def parse_kp_page(html):
|
||
"""考点详情页: 逐 article 提取 年份/题号/题型/题干/选项/答案/解析"""
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
out = []
|
||
for art in soup.find_all("article"):
|
||
head = art.find("p", class_="font-num")
|
||
if not head:
|
||
continue
|
||
ah = head.find("a", href=re.compile(r"^/zhenti/\d{4}"))
|
||
if not ah:
|
||
continue
|
||
mm = re.match(r"/zhenti/(\d{4})(?:#q(\d+)|/(\d+))", ah["href"])
|
||
if not mm:
|
||
continue
|
||
year, qno = int(mm.group(1)), int(mm.group(2) or mm.group(3))
|
||
head_text = head.get_text(" ", strip=True)
|
||
qtype = "多选" if "多选" in head_text else ("单选" if "单选" in head_text else "")
|
||
|
||
stem_el = art.find("p", class_=re.compile(r"mt-1\.5"))
|
||
stem = stem_el.get_text(" ", strip=True) if stem_el else ""
|
||
|
||
opts = []
|
||
for p in art.find_all("p", class_="stopt"):
|
||
t = p.get_text("", strip=True).replace("✓", "").replace("✗", "").strip()
|
||
mopt = re.match(r"^([A-D])\s*[..、]\s*(.*)$", t)
|
||
if mopt:
|
||
opts.append({"letter": mopt.group(1), "text": mopt.group(2).strip(),
|
||
"correct": p.get("data-ok") == "1"})
|
||
else:
|
||
opts.append({"letter": "", "text": t, "correct": p.get("data-ok") == "1"})
|
||
|
||
ansbox = art.find("div", class_="ansbox")
|
||
answer = analysis = note = ""
|
||
if ansbox:
|
||
b = ansbox.find("b")
|
||
if b:
|
||
answer = b.get_text(strip=True).replace("答案", "").strip()
|
||
full = ansbox.get_text("\n", strip=True)
|
||
full = re.sub(r"^答案\s*[A-D]+\s*", "", full).strip()
|
||
nm = re.search(r"注:.*?(?=\n|$)", full)
|
||
if nm:
|
||
note = nm.group(0).strip()
|
||
full = full.replace(note, "").strip()
|
||
analysis = re.sub(r"\n{2,}", "\n", full).strip()
|
||
|
||
out.append({"year": year, "qno": qno, "type": qtype, "stem": stem,
|
||
"options": opts, "answer": answer, "analysis": analysis,
|
||
"note": note, "url": BASE + f"/zhenti/{year}/{qno}"})
|
||
return out
|
||
|
||
|
||
def main():
|
||
os.makedirs(HERE, exist_ok=True)
|
||
con = sqlite3.connect(DB)
|
||
init_db(con)
|
||
cur = con.cursor()
|
||
|
||
idx = get(INDEX)
|
||
if not idx:
|
||
print("索引页抓取失败", flush=True)
|
||
return 1
|
||
items = parse_index(idx)
|
||
print(f"索引解析: {len(items)} 个考点", flush=True)
|
||
|
||
cur.execute("SELECT name, crawled_count FROM kaodian")
|
||
done = {r[0]: r[1] for r in cur.fetchall()}
|
||
|
||
for i, it in enumerate(items, 1):
|
||
if it["name"] in done and done[it["name"]] > 0:
|
||
print(f"[{i}/{len(items)}] 跳过(已爬{done[it['name']]}题): {it['name']}", flush=True)
|
||
continue
|
||
time.sleep(BASE_DELAY + random.random() * 0.3) # 礼貌限速
|
||
html = get(it["url"])
|
||
if not html:
|
||
print(f"[{i}/{len(items)}] 失败: {it['name']}", flush=True)
|
||
continue
|
||
qs = parse_kp_page(html)
|
||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||
cur.execute("SELECT id FROM kaodian WHERE name=?", (it["name"],))
|
||
row = cur.fetchone()
|
||
if row:
|
||
kid = row[0]
|
||
cur.execute("UPDATE kaodian SET qcount=?, url=?, crawled_count=?, crawled_at=? WHERE id=?",
|
||
(it["qcount"], it["url"], len(qs), now, kid))
|
||
else:
|
||
cur.execute("INSERT INTO kaodian(subject,name,qcount,url,crawled_count,crawled_at) VALUES(?,?,?,?,?,?)",
|
||
(it["subject"], it["name"], it["qcount"], it["url"], len(qs), now))
|
||
kid = cur.lastrowid
|
||
for q in qs:
|
||
cur.execute("INSERT OR REPLACE INTO zhenti(kaodian_id,year,qno,qtype,stem,options,answer,analysis,note,url) "
|
||
"VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||
(kid, q["year"], q["qno"], q["type"], q["stem"],
|
||
str(q["options"]), q["answer"], q["analysis"], q["note"], q["url"]))
|
||
con.commit()
|
||
tag = "✓" if len(qs) == it["qcount"] else f"⚠({len(qs)}/{it['qcount']})"
|
||
print(f"[{i}/{len(items)}] {tag} {it['subject']} | {it['name']}", flush=True)
|
||
|
||
cur.execute("SELECT COUNT(*) FROM zhenti")
|
||
n = cur.fetchone()[0]
|
||
cur.execute("SELECT COUNT(*) FROM kaodian WHERE crawled_count>0")
|
||
k = cur.fetchone()[0]
|
||
print(f"完成: 入库 {k} 个考点, zhenti 表共 {n} 题", flush=True)
|
||
con.close()
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|