真题入库SQLite(544题)+马原9专题内嵌真题精练(134题),修复双链别名分隔符,规范更新
This commit is contained in:
@@ -0,0 +1,193 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""真题营·考研政治考点真题爬虫 → SQLite
|
||||
来源: https://zhenti.zalize.com/zhenti/kaodian (2010-2026 按官方考点整理的历年真题, 含答案与解析)
|
||||
礼貌爬取: 每请求间隔 0.5~0.8s 随机抖动; 失败指数退避重试(1s/2s/4s, 3次); 断点续爬(已入库考点跳过)。
|
||||
用法: python crawl_zhenti.py (需设置 HTTP_PROXY/HTTPS_PROXY 环境变量走代理)
|
||||
"""
|
||||
import os, re, sys, time, random, sqlite3, urllib.parse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
BASE = "https://zhenti.zalize.com"
|
||||
INDEX = BASE + "/zhenti/kaodian"
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DB = os.path.join(HERE, "zhenti_kaodian.db")
|
||||
BASE_DELAY = 0.5 # 秒, 每次请求前随机 +0~0.3s
|
||||
|
||||
|
||||
def get(url, retries=3):
|
||||
for i in range(retries):
|
||||
try:
|
||||
r = requests.get(url, headers=HEADERS, timeout=30)
|
||||
if r.status_code == 200:
|
||||
return r.text
|
||||
print(f" HTTP {r.status_code} {url}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" ERR {type(e).__name__} {url}", flush=True)
|
||||
time.sleep(2 ** i) # 1s, 2s, 4s 退避
|
||||
return None
|
||||
|
||||
|
||||
def init_db(con):
|
||||
con.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS kaodian(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
subject TEXT NOT NULL,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
qcount INTEGER, -- 站点标注的真题数
|
||||
url TEXT,
|
||||
crawled_count INTEGER DEFAULT 0,
|
||||
crawled_at TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS zhenti(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
kaodian_id INTEGER NOT NULL REFERENCES kaodian(id),
|
||||
year INTEGER,
|
||||
qno INTEGER,
|
||||
qtype TEXT, -- 单选 / 多选
|
||||
stem TEXT, -- 题干
|
||||
options TEXT, -- JSON: [{"letter","text","correct"}]
|
||||
answer TEXT, -- 如 ABD
|
||||
analysis TEXT, -- 解析全文
|
||||
note TEXT, -- 解析内的争议/备注(如"各机构答案存在分歧")
|
||||
url TEXT,
|
||||
UNIQUE(kaodian_id, year, qno)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_zhenti_kp ON zhenti(kaodian_id);
|
||||
""")
|
||||
con.commit()
|
||||
|
||||
|
||||
def parse_index(html):
|
||||
"""索引页: 提取 科目→考点名→题量→详情URL"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
items = []
|
||||
for h2 in soup.find_all("h2", id=re.compile(r"^s\d")):
|
||||
subject = h2.contents[0].strip() if h2.contents else ""
|
||||
div = h2.find_next_sibling("div")
|
||||
if not div:
|
||||
continue
|
||||
for a in div.find_all("a", href=re.compile(r"^/zhenti/kaodian/")):
|
||||
label = a.get_text(" ", strip=True)
|
||||
m = re.search(r"(\d+)\s*$", label)
|
||||
qc = int(m.group(1)) if m else 0
|
||||
name = re.sub(r"\s*\d+\s*$", "", label).strip()
|
||||
items.append({"subject": subject, "name": name, "qcount": qc,
|
||||
"url": BASE + a["href"]})
|
||||
return items
|
||||
|
||||
|
||||
def parse_kp_page(html):
|
||||
"""考点详情页: 逐 article 提取 年份/题号/题型/题干/选项/答案/解析"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
out = []
|
||||
for art in soup.find_all("article"):
|
||||
head = art.find("p", class_="font-num")
|
||||
if not head:
|
||||
continue
|
||||
ah = head.find("a", href=re.compile(r"^/zhenti/\d{4}"))
|
||||
if not ah:
|
||||
continue
|
||||
mm = re.match(r"/zhenti/(\d{4})(?:#q(\d+)|/(\d+))", ah["href"])
|
||||
if not mm:
|
||||
continue
|
||||
year, qno = int(mm.group(1)), int(mm.group(2) or mm.group(3))
|
||||
head_text = head.get_text(" ", strip=True)
|
||||
qtype = "多选" if "多选" in head_text else ("单选" if "单选" in head_text else "")
|
||||
|
||||
stem_el = art.find("p", class_=re.compile(r"mt-1\.5"))
|
||||
stem = stem_el.get_text(" ", strip=True) if stem_el else ""
|
||||
|
||||
opts = []
|
||||
for p in art.find_all("p", class_="stopt"):
|
||||
t = p.get_text("", strip=True).replace("✓", "").replace("✗", "").strip()
|
||||
mopt = re.match(r"^([A-D])\s*[..、]\s*(.*)$", t)
|
||||
if mopt:
|
||||
opts.append({"letter": mopt.group(1), "text": mopt.group(2).strip(),
|
||||
"correct": p.get("data-ok") == "1"})
|
||||
else:
|
||||
opts.append({"letter": "", "text": t, "correct": p.get("data-ok") == "1"})
|
||||
|
||||
ansbox = art.find("div", class_="ansbox")
|
||||
answer = analysis = note = ""
|
||||
if ansbox:
|
||||
b = ansbox.find("b")
|
||||
if b:
|
||||
answer = b.get_text(strip=True).replace("答案", "").strip()
|
||||
full = ansbox.get_text("\n", strip=True)
|
||||
full = re.sub(r"^答案\s*[A-D]+\s*", "", full).strip()
|
||||
nm = re.search(r"注:.*?(?=\n|$)", full)
|
||||
if nm:
|
||||
note = nm.group(0).strip()
|
||||
full = full.replace(note, "").strip()
|
||||
analysis = re.sub(r"\n{2,}", "\n", full).strip()
|
||||
|
||||
out.append({"year": year, "qno": qno, "type": qtype, "stem": stem,
|
||||
"options": opts, "answer": answer, "analysis": analysis,
|
||||
"note": note, "url": BASE + f"/zhenti/{year}/{qno}"})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(HERE, exist_ok=True)
|
||||
con = sqlite3.connect(DB)
|
||||
init_db(con)
|
||||
cur = con.cursor()
|
||||
|
||||
idx = get(INDEX)
|
||||
if not idx:
|
||||
print("索引页抓取失败", flush=True)
|
||||
return 1
|
||||
items = parse_index(idx)
|
||||
print(f"索引解析: {len(items)} 个考点", flush=True)
|
||||
|
||||
cur.execute("SELECT name, crawled_count FROM kaodian")
|
||||
done = {r[0]: r[1] for r in cur.fetchall()}
|
||||
|
||||
for i, it in enumerate(items, 1):
|
||||
if it["name"] in done and done[it["name"]] > 0:
|
||||
print(f"[{i}/{len(items)}] 跳过(已爬{done[it['name']]}题): {it['name']}", flush=True)
|
||||
continue
|
||||
time.sleep(BASE_DELAY + random.random() * 0.3) # 礼貌限速
|
||||
html = get(it["url"])
|
||||
if not html:
|
||||
print(f"[{i}/{len(items)}] 失败: {it['name']}", flush=True)
|
||||
continue
|
||||
qs = parse_kp_page(html)
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
cur.execute("SELECT id FROM kaodian WHERE name=?", (it["name"],))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
kid = row[0]
|
||||
cur.execute("UPDATE kaodian SET qcount=?, url=?, crawled_count=?, crawled_at=? WHERE id=?",
|
||||
(it["qcount"], it["url"], len(qs), now, kid))
|
||||
else:
|
||||
cur.execute("INSERT INTO kaodian(subject,name,qcount,url,crawled_count,crawled_at) VALUES(?,?,?,?,?,?)",
|
||||
(it["subject"], it["name"], it["qcount"], it["url"], len(qs), now))
|
||||
kid = cur.lastrowid
|
||||
for q in qs:
|
||||
cur.execute("INSERT OR REPLACE INTO zhenti(kaodian_id,year,qno,qtype,stem,options,answer,analysis,note,url) "
|
||||
"VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(kid, q["year"], q["qno"], q["type"], q["stem"],
|
||||
str(q["options"]), q["answer"], q["analysis"], q["note"], q["url"]))
|
||||
con.commit()
|
||||
tag = "✓" if len(qs) == it["qcount"] else f"⚠({len(qs)}/{it['qcount']})"
|
||||
print(f"[{i}/{len(items)}] {tag} {it['subject']} | {it['name']}", flush=True)
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM zhenti")
|
||||
n = cur.fetchone()[0]
|
||||
cur.execute("SELECT COUNT(*) FROM kaodian WHERE crawled_count>0")
|
||||
k = cur.fetchone()[0]
|
||||
print(f"完成: 入库 {k} 个考点, zhenti 表共 {n} 题", flush=True)
|
||||
con.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user