真题入库SQLite(544题)+马原9专题内嵌真题精练(134题),修复双链别名分隔符,规范更新

This commit is contained in:
2026-10-26 22:29:07 +08:00
parent b5d451a49d
commit 8f2391b16d
22 changed files with 1448 additions and 54 deletions
+193
View File
@@ -0,0 +1,193 @@
# -*- coding: utf-8 -*-
"""真题营·考研政治考点真题爬虫 → SQLite
来源: https://zhenti.zalize.com/zhenti/kaodian (2010-2026 按官方考点整理的历年真题, 含答案与解析)
礼貌爬取: 每请求间隔 0.5~0.8s 随机抖动; 失败指数退避重试(1s/2s/4s, 3次); 断点续爬(已入库考点跳过)。
用法: python crawl_zhenti.py (需设置 HTTP_PROXY/HTTPS_PROXY 环境变量走代理)
"""
import os, re, sys, time, random, sqlite3, urllib.parse
import requests
from bs4 import BeautifulSoup
BASE = "https://zhenti.zalize.com"
INDEX = BASE + "/zhenti/kaodian"
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9",
}
HERE = os.path.dirname(os.path.abspath(__file__))
DB = os.path.join(HERE, "zhenti_kaodian.db")
BASE_DELAY = 0.5 # 秒, 每次请求前随机 +0~0.3s
def get(url, retries=3):
for i in range(retries):
try:
r = requests.get(url, headers=HEADERS, timeout=30)
if r.status_code == 200:
return r.text
print(f" HTTP {r.status_code} {url}", flush=True)
except Exception as e:
print(f" ERR {type(e).__name__} {url}", flush=True)
time.sleep(2 ** i) # 1s, 2s, 4s 退避
return None
def init_db(con):
con.executescript("""
CREATE TABLE IF NOT EXISTS kaodian(
id INTEGER PRIMARY KEY AUTOINCREMENT,
subject TEXT NOT NULL,
name TEXT NOT NULL UNIQUE,
qcount INTEGER, -- 站点标注的真题数
url TEXT,
crawled_count INTEGER DEFAULT 0,
crawled_at TEXT
);
CREATE TABLE IF NOT EXISTS zhenti(
id INTEGER PRIMARY KEY AUTOINCREMENT,
kaodian_id INTEGER NOT NULL REFERENCES kaodian(id),
year INTEGER,
qno INTEGER,
qtype TEXT, -- 单选 / 多选
stem TEXT, -- 题干
options TEXT, -- JSON: [{"letter","text","correct"}]
answer TEXT, -- 如 ABD
analysis TEXT, -- 解析全文
note TEXT, -- 解析内的争议/备注(如"各机构答案存在分歧")
url TEXT,
UNIQUE(kaodian_id, year, qno)
);
CREATE INDEX IF NOT EXISTS idx_zhenti_kp ON zhenti(kaodian_id);
""")
con.commit()
def parse_index(html):
"""索引页: 提取 科目→考点名→题量→详情URL"""
soup = BeautifulSoup(html, "html.parser")
items = []
for h2 in soup.find_all("h2", id=re.compile(r"^s\d")):
subject = h2.contents[0].strip() if h2.contents else ""
div = h2.find_next_sibling("div")
if not div:
continue
for a in div.find_all("a", href=re.compile(r"^/zhenti/kaodian/")):
label = a.get_text(" ", strip=True)
m = re.search(r"(\d+)\s*$", label)
qc = int(m.group(1)) if m else 0
name = re.sub(r"\s*\d+\s*$", "", label).strip()
items.append({"subject": subject, "name": name, "qcount": qc,
"url": BASE + a["href"]})
return items
def parse_kp_page(html):
"""考点详情页: 逐 article 提取 年份/题号/题型/题干/选项/答案/解析"""
soup = BeautifulSoup(html, "html.parser")
out = []
for art in soup.find_all("article"):
head = art.find("p", class_="font-num")
if not head:
continue
ah = head.find("a", href=re.compile(r"^/zhenti/\d{4}"))
if not ah:
continue
mm = re.match(r"/zhenti/(\d{4})(?:#q(\d+)|/(\d+))", ah["href"])
if not mm:
continue
year, qno = int(mm.group(1)), int(mm.group(2) or mm.group(3))
head_text = head.get_text(" ", strip=True)
qtype = "多选" if "多选" in head_text else ("单选" if "单选" in head_text else "")
stem_el = art.find("p", class_=re.compile(r"mt-1\.5"))
stem = stem_el.get_text(" ", strip=True) if stem_el else ""
opts = []
for p in art.find_all("p", class_="stopt"):
t = p.get_text("", strip=True).replace("✓", "").replace("✗", "").strip()
mopt = re.match(r"^([A-D])\s*[..、]\s*(.*)$", t)
if mopt:
opts.append({"letter": mopt.group(1), "text": mopt.group(2).strip(),
"correct": p.get("data-ok") == "1"})
else:
opts.append({"letter": "", "text": t, "correct": p.get("data-ok") == "1"})
ansbox = art.find("div", class_="ansbox")
answer = analysis = note = ""
if ansbox:
b = ansbox.find("b")
if b:
answer = b.get_text(strip=True).replace("答案", "").strip()
full = ansbox.get_text("\n", strip=True)
full = re.sub(r"^答案\s*[A-D]+\s*", "", full).strip()
nm = re.search(r"注:.*?(?=\n|$)", full)
if nm:
note = nm.group(0).strip()
full = full.replace(note, "").strip()
analysis = re.sub(r"\n{2,}", "\n", full).strip()
out.append({"year": year, "qno": qno, "type": qtype, "stem": stem,
"options": opts, "answer": answer, "analysis": analysis,
"note": note, "url": BASE + f"/zhenti/{year}/{qno}"})
return out
def main():
os.makedirs(HERE, exist_ok=True)
con = sqlite3.connect(DB)
init_db(con)
cur = con.cursor()
idx = get(INDEX)
if not idx:
print("索引页抓取失败", flush=True)
return 1
items = parse_index(idx)
print(f"索引解析: {len(items)} 个考点", flush=True)
cur.execute("SELECT name, crawled_count FROM kaodian")
done = {r[0]: r[1] for r in cur.fetchall()}
for i, it in enumerate(items, 1):
if it["name"] in done and done[it["name"]] > 0:
print(f"[{i}/{len(items)}] 跳过(已爬{done[it['name']]}题): {it['name']}", flush=True)
continue
time.sleep(BASE_DELAY + random.random() * 0.3) # 礼貌限速
html = get(it["url"])
if not html:
print(f"[{i}/{len(items)}] 失败: {it['name']}", flush=True)
continue
qs = parse_kp_page(html)
now = time.strftime("%Y-%m-%d %H:%M:%S")
cur.execute("SELECT id FROM kaodian WHERE name=?", (it["name"],))
row = cur.fetchone()
if row:
kid = row[0]
cur.execute("UPDATE kaodian SET qcount=?, url=?, crawled_count=?, crawled_at=? WHERE id=?",
(it["qcount"], it["url"], len(qs), now, kid))
else:
cur.execute("INSERT INTO kaodian(subject,name,qcount,url,crawled_count,crawled_at) VALUES(?,?,?,?,?,?)",
(it["subject"], it["name"], it["qcount"], it["url"], len(qs), now))
kid = cur.lastrowid
for q in qs:
cur.execute("INSERT OR REPLACE INTO zhenti(kaodian_id,year,qno,qtype,stem,options,answer,analysis,note,url) "
"VALUES(?,?,?,?,?,?,?,?,?,?)",
(kid, q["year"], q["qno"], q["type"], q["stem"],
str(q["options"]), q["answer"], q["analysis"], q["note"], q["url"]))
con.commit()
tag = "✓" if len(qs) == it["qcount"] else f"⚠({len(qs)}/{it['qcount']})"
print(f"[{i}/{len(items)}] {tag} {it['subject']} | {it['name']}", flush=True)
cur.execute("SELECT COUNT(*) FROM zhenti")
n = cur.fetchone()[0]
cur.execute("SELECT COUNT(*) FROM kaodian WHERE crawled_count>0")
k = cur.fetchone()[0]
print(f"完成: 入库 {k} 个考点, zhenti 表共 {n} 题", flush=True)
con.close()
return 0
if __name__ == "__main__":
sys.exit(main())
+143
View File
@@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
"""把 SQLite 题库中的真题按考点生成「真题精练」区,插入马原各专题文件。
映射:真题营考点名 → 马原专题号(对齐 马克思主义基本原理.md 的 9 个专题)。
插入位置:每专题文件 "## 三、易混点速查" 之前。
用法:python generate_zhenti_sections.py
"""
import os, re, sqlite3, json, ast
HERE = os.path.dirname(os.path.abspath(__file__))
DB = os.path.join(HERE, "zhenti_kaodian.db")
NOTE_DIR = r"D:\考研\考研政治\政治笔记\马原"
# 真题营考点名 → 马原专题号
TOPIC_MAP = {
"马克思主义的创立与发展": 1,
"主观能动性与客观规律性": 2, "物质与意识的辩证关系": 2, "世界的物质统一性": 2, "运动与静止": 2,
"联系与发展": 3, "必然性与偶然性": 3, "对立统一规律": 3, "量变质变规律": 3,
"矛盾的普遍性与特殊性": 3, "否定之否定规律": 3,
"实践是认识的基础": 4, "真理与价值": 4, "感性认识与理性认识": 4,
"认识的反复性与无限性": 4, "认识的两次飞跃": 4,
"社会存在与社会意识": 5, "经济基础与上层建筑": 5, "生产力与生产关系": 5,
"人民群众是历史的创造者": 5, "个人在历史上的作用": 5, "社会形态更替": 5, "社会基本矛盾": 5,
"商品二因素与劳动二重性": 6,
"剩余价值的生产": 7, "资本的有机构成": 7, "资本主义经济危机": 7,
"垄断资本主义与金融资本": 8,
"共产主义社会的基本特征": 9, "无产阶级革命与马克思主义政党": 9, "两个必然与两个决不会": 9,
}
SUBJ_PAT = "马原"
def load_questions():
con = sqlite3.connect(DB)
con.row_factory = sqlite3.Row
cur = con.cursor()
rows = cur.execute(
"SELECT k.name AS kp, k.subject, z.* FROM zhenti z JOIN kaodian k ON k.id=z.kaodian_id"
).fetchall()
con.close()
qs = []
for r in rows:
if SUBJ_PAT not in r["subject"]:
continue
opts = ast.literal_eval(r["options"]) if r["options"] else []
qs.append({
"kp": r["kp"], "year": r["year"], "qno": r["qno"], "type": r["qtype"],
"stem": r["stem"], "opts": opts, "answer": r["answer"],
"analysis": r["analysis"] or "", "note": r["note"] or "", "url": r["url"],
})
# 去重:同一 年份+题号+考点 只留一条(跨科目重复考点会重复出现)
seen, uniq = set(), []
for q in sorted(qs, key=lambda x: (x["year"], x["qno"])):
key = (q["year"], q["qno"], q["kp"])
if key in seen:
continue
seen.add(key)
uniq.append(q)
return uniq
def fmt_options(opts):
return " ".join(f"{o['letter']}. {o['text']}" for o in opts)
def fmt_analysis(a, limit=240):
a = re.sub(r"\n+", " ", a).strip()
if len(a) > limit:
return a[:limit] + "……"
return a
def build_section(qs):
if not qs:
return ""
lines = [
"### 真题精练(真题营 2010—2026 全量真题)",
"",
"> 来源:真题营 https://zhenti.zalize.com/zhenti/kaodian(2010—2026 历年真题,含答案与解析;已入库 SQLite 题库 `题库/zhenti_kaodian.db`,共 544 题)。以下收录本专题对应考点的全部历年真题,按年份排序,可直接闭卷刷。",
"",
]
for i, q in enumerate(qs, 1):
qtype = "多选" if q["type"] == "多选" else "单选"
lines.append(f"**{i}. {q['year']}-{q['qno']} {qtype} · {q['kp']}**")
lines.append(q["stem"])
if q["opts"]:
lines.append(fmt_options(q["opts"]))
note = f" {q['note']}" if q["note"] else ""
lines.append(f"**答案 {q['answer']}**{note}|**解析**:{fmt_analysis(q['analysis'])}")
lines.append(f"> 原题出处:{q['url']}")
lines.append("")
return "\n".join(lines)
def main():
qs = load_questions()
by_topic = {}
for q in qs:
t = TOPIC_MAP.get(q["kp"])
if t is None:
print(f"!! 未映射考点: {q['kp']}", flush=True)
continue
by_topic.setdefault(t, []).append(q)
report = []
total_chars = 0
for t in range(1, 10):
path = os.path.join(NOTE_DIR, f"专题{t} 某某.md")
# 实际文件名按目录列出
# 直接用 glob 找文件
import glob
files = {int(m.group(1)): m.group(0) for m in
(re.match(r"专题(\d+) (.+?)\.md", os.path.basename(p)) for p in
glob.glob(os.path.join(NOTE_DIR, "专题*.md"))) if m}
for t in sorted(files):
fname = files[t]
path = os.path.join(NOTE_DIR, fname)
text = open(path, encoding="utf-8").read()
sec = build_section(by_topic.get(t, []))
if not sec:
report.append(f"{fname}: 无真题,跳过")
continue
anchor = "## 三、易混点速查"
if anchor not in text:
report.append(f"{fname}: 未找到锚点 {anchor},跳过")
continue
# 幂等: 移除已存在的旧真题精练区再重建
m = re.search(r"### 真题精练.*?(?=\n## 三、)", text, re.S)
if m:
text = text[: m.start()] + text[m.end():]
text = text.replace(anchor, sec + "\n\n" + anchor, 1)
open(path, "w", encoding="utf-8").write(text)
nq = len(by_topic.get(t, []))
nchars = len(text)
total_chars += nchars
report.append(f"{fname}: 插入 {nq} 题,文件 {nchars} 字符")
report.append(f"马原 9 专题共 {len(qs)} 真题,合计约 {total_chars} 字符")
out = os.path.join(HERE, "_gen_report.txt")
open(out, "w", encoding="utf-8").write("\n".join(report))
print("\n".join(report), flush=True)
if __name__ == "__main__":
main()
Binary file not shown.