真题入库SQLite(544题)+马原9专题内嵌真题精练(134题),修复双链别名分隔符,规范更新
This commit is contained in:
@@ -0,0 +1,193 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""真题营·考研政治考点真题爬虫 → SQLite
|
||||
来源: https://zhenti.zalize.com/zhenti/kaodian (2010-2026 按官方考点整理的历年真题, 含答案与解析)
|
||||
礼貌爬取: 每请求间隔 0.5~0.8s 随机抖动; 失败指数退避重试(1s/2s/4s, 3次); 断点续爬(已入库考点跳过)。
|
||||
用法: python crawl_zhenti.py (需设置 HTTP_PROXY/HTTPS_PROXY 环境变量走代理)
|
||||
"""
|
||||
import os, re, sys, time, random, sqlite3, urllib.parse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
BASE = "https://zhenti.zalize.com"
|
||||
INDEX = BASE + "/zhenti/kaodian"
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DB = os.path.join(HERE, "zhenti_kaodian.db")
|
||||
BASE_DELAY = 0.5 # 秒, 每次请求前随机 +0~0.3s
|
||||
|
||||
|
||||
def get(url, retries=3):
|
||||
for i in range(retries):
|
||||
try:
|
||||
r = requests.get(url, headers=HEADERS, timeout=30)
|
||||
if r.status_code == 200:
|
||||
return r.text
|
||||
print(f" HTTP {r.status_code} {url}", flush=True)
|
||||
except Exception as e:
|
||||
print(f" ERR {type(e).__name__} {url}", flush=True)
|
||||
time.sleep(2 ** i) # 1s, 2s, 4s 退避
|
||||
return None
|
||||
|
||||
|
||||
def init_db(con):
|
||||
con.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS kaodian(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
subject TEXT NOT NULL,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
qcount INTEGER, -- 站点标注的真题数
|
||||
url TEXT,
|
||||
crawled_count INTEGER DEFAULT 0,
|
||||
crawled_at TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS zhenti(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
kaodian_id INTEGER NOT NULL REFERENCES kaodian(id),
|
||||
year INTEGER,
|
||||
qno INTEGER,
|
||||
qtype TEXT, -- 单选 / 多选
|
||||
stem TEXT, -- 题干
|
||||
options TEXT, -- JSON: [{"letter","text","correct"}]
|
||||
answer TEXT, -- 如 ABD
|
||||
analysis TEXT, -- 解析全文
|
||||
note TEXT, -- 解析内的争议/备注(如"各机构答案存在分歧")
|
||||
url TEXT,
|
||||
UNIQUE(kaodian_id, year, qno)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_zhenti_kp ON zhenti(kaodian_id);
|
||||
""")
|
||||
con.commit()
|
||||
|
||||
|
||||
def parse_index(html):
|
||||
"""索引页: 提取 科目→考点名→题量→详情URL"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
items = []
|
||||
for h2 in soup.find_all("h2", id=re.compile(r"^s\d")):
|
||||
subject = h2.contents[0].strip() if h2.contents else ""
|
||||
div = h2.find_next_sibling("div")
|
||||
if not div:
|
||||
continue
|
||||
for a in div.find_all("a", href=re.compile(r"^/zhenti/kaodian/")):
|
||||
label = a.get_text(" ", strip=True)
|
||||
m = re.search(r"(\d+)\s*$", label)
|
||||
qc = int(m.group(1)) if m else 0
|
||||
name = re.sub(r"\s*\d+\s*$", "", label).strip()
|
||||
items.append({"subject": subject, "name": name, "qcount": qc,
|
||||
"url": BASE + a["href"]})
|
||||
return items
|
||||
|
||||
|
||||
def parse_kp_page(html):
|
||||
"""考点详情页: 逐 article 提取 年份/题号/题型/题干/选项/答案/解析"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
out = []
|
||||
for art in soup.find_all("article"):
|
||||
head = art.find("p", class_="font-num")
|
||||
if not head:
|
||||
continue
|
||||
ah = head.find("a", href=re.compile(r"^/zhenti/\d{4}"))
|
||||
if not ah:
|
||||
continue
|
||||
mm = re.match(r"/zhenti/(\d{4})(?:#q(\d+)|/(\d+))", ah["href"])
|
||||
if not mm:
|
||||
continue
|
||||
year, qno = int(mm.group(1)), int(mm.group(2) or mm.group(3))
|
||||
head_text = head.get_text(" ", strip=True)
|
||||
qtype = "多选" if "多选" in head_text else ("单选" if "单选" in head_text else "")
|
||||
|
||||
stem_el = art.find("p", class_=re.compile(r"mt-1\.5"))
|
||||
stem = stem_el.get_text(" ", strip=True) if stem_el else ""
|
||||
|
||||
opts = []
|
||||
for p in art.find_all("p", class_="stopt"):
|
||||
t = p.get_text("", strip=True).replace("✓", "").replace("✗", "").strip()
|
||||
mopt = re.match(r"^([A-D])\s*[..、]\s*(.*)$", t)
|
||||
if mopt:
|
||||
opts.append({"letter": mopt.group(1), "text": mopt.group(2).strip(),
|
||||
"correct": p.get("data-ok") == "1"})
|
||||
else:
|
||||
opts.append({"letter": "", "text": t, "correct": p.get("data-ok") == "1"})
|
||||
|
||||
ansbox = art.find("div", class_="ansbox")
|
||||
answer = analysis = note = ""
|
||||
if ansbox:
|
||||
b = ansbox.find("b")
|
||||
if b:
|
||||
answer = b.get_text(strip=True).replace("答案", "").strip()
|
||||
full = ansbox.get_text("\n", strip=True)
|
||||
full = re.sub(r"^答案\s*[A-D]+\s*", "", full).strip()
|
||||
nm = re.search(r"注:.*?(?=\n|$)", full)
|
||||
if nm:
|
||||
note = nm.group(0).strip()
|
||||
full = full.replace(note, "").strip()
|
||||
analysis = re.sub(r"\n{2,}", "\n", full).strip()
|
||||
|
||||
out.append({"year": year, "qno": qno, "type": qtype, "stem": stem,
|
||||
"options": opts, "answer": answer, "analysis": analysis,
|
||||
"note": note, "url": BASE + f"/zhenti/{year}/{qno}"})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(HERE, exist_ok=True)
|
||||
con = sqlite3.connect(DB)
|
||||
init_db(con)
|
||||
cur = con.cursor()
|
||||
|
||||
idx = get(INDEX)
|
||||
if not idx:
|
||||
print("索引页抓取失败", flush=True)
|
||||
return 1
|
||||
items = parse_index(idx)
|
||||
print(f"索引解析: {len(items)} 个考点", flush=True)
|
||||
|
||||
cur.execute("SELECT name, crawled_count FROM kaodian")
|
||||
done = {r[0]: r[1] for r in cur.fetchall()}
|
||||
|
||||
for i, it in enumerate(items, 1):
|
||||
if it["name"] in done and done[it["name"]] > 0:
|
||||
print(f"[{i}/{len(items)}] 跳过(已爬{done[it['name']]}题): {it['name']}", flush=True)
|
||||
continue
|
||||
time.sleep(BASE_DELAY + random.random() * 0.3) # 礼貌限速
|
||||
html = get(it["url"])
|
||||
if not html:
|
||||
print(f"[{i}/{len(items)}] 失败: {it['name']}", flush=True)
|
||||
continue
|
||||
qs = parse_kp_page(html)
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
cur.execute("SELECT id FROM kaodian WHERE name=?", (it["name"],))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
kid = row[0]
|
||||
cur.execute("UPDATE kaodian SET qcount=?, url=?, crawled_count=?, crawled_at=? WHERE id=?",
|
||||
(it["qcount"], it["url"], len(qs), now, kid))
|
||||
else:
|
||||
cur.execute("INSERT INTO kaodian(subject,name,qcount,url,crawled_count,crawled_at) VALUES(?,?,?,?,?,?)",
|
||||
(it["subject"], it["name"], it["qcount"], it["url"], len(qs), now))
|
||||
kid = cur.lastrowid
|
||||
for q in qs:
|
||||
cur.execute("INSERT OR REPLACE INTO zhenti(kaodian_id,year,qno,qtype,stem,options,answer,analysis,note,url) "
|
||||
"VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(kid, q["year"], q["qno"], q["type"], q["stem"],
|
||||
str(q["options"]), q["answer"], q["analysis"], q["note"], q["url"]))
|
||||
con.commit()
|
||||
tag = "✓" if len(qs) == it["qcount"] else f"⚠({len(qs)}/{it['qcount']})"
|
||||
print(f"[{i}/{len(items)}] {tag} {it['subject']} | {it['name']}", flush=True)
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM zhenti")
|
||||
n = cur.fetchone()[0]
|
||||
cur.execute("SELECT COUNT(*) FROM kaodian WHERE crawled_count>0")
|
||||
k = cur.fetchone()[0]
|
||||
print(f"完成: 入库 {k} 个考点, zhenti 表共 {n} 题", flush=True)
|
||||
con.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,143 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""把 SQLite 题库中的真题按考点生成「真题精练」区,插入马原各专题文件。
|
||||
映射:真题营考点名 → 马原专题号(对齐 马克思主义基本原理.md 的 9 个专题)。
|
||||
插入位置:每专题文件 "## 三、易混点速查" 之前。
|
||||
用法:python generate_zhenti_sections.py
|
||||
"""
|
||||
import os, re, sqlite3, json, ast
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DB = os.path.join(HERE, "zhenti_kaodian.db")
|
||||
NOTE_DIR = r"D:\考研\考研政治\政治笔记\马原"
|
||||
|
||||
# 真题营考点名 → 马原专题号
|
||||
TOPIC_MAP = {
|
||||
"马克思主义的创立与发展": 1,
|
||||
"主观能动性与客观规律性": 2, "物质与意识的辩证关系": 2, "世界的物质统一性": 2, "运动与静止": 2,
|
||||
"联系与发展": 3, "必然性与偶然性": 3, "对立统一规律": 3, "量变质变规律": 3,
|
||||
"矛盾的普遍性与特殊性": 3, "否定之否定规律": 3,
|
||||
"实践是认识的基础": 4, "真理与价值": 4, "感性认识与理性认识": 4,
|
||||
"认识的反复性与无限性": 4, "认识的两次飞跃": 4,
|
||||
"社会存在与社会意识": 5, "经济基础与上层建筑": 5, "生产力与生产关系": 5,
|
||||
"人民群众是历史的创造者": 5, "个人在历史上的作用": 5, "社会形态更替": 5, "社会基本矛盾": 5,
|
||||
"商品二因素与劳动二重性": 6,
|
||||
"剩余价值的生产": 7, "资本的有机构成": 7, "资本主义经济危机": 7,
|
||||
"垄断资本主义与金融资本": 8,
|
||||
"共产主义社会的基本特征": 9, "无产阶级革命与马克思主义政党": 9, "两个必然与两个决不会": 9,
|
||||
}
|
||||
|
||||
SUBJ_PAT = "马原"
|
||||
|
||||
|
||||
def load_questions():
|
||||
con = sqlite3.connect(DB)
|
||||
con.row_factory = sqlite3.Row
|
||||
cur = con.cursor()
|
||||
rows = cur.execute(
|
||||
"SELECT k.name AS kp, k.subject, z.* FROM zhenti z JOIN kaodian k ON k.id=z.kaodian_id"
|
||||
).fetchall()
|
||||
con.close()
|
||||
qs = []
|
||||
for r in rows:
|
||||
if SUBJ_PAT not in r["subject"]:
|
||||
continue
|
||||
opts = ast.literal_eval(r["options"]) if r["options"] else []
|
||||
qs.append({
|
||||
"kp": r["kp"], "year": r["year"], "qno": r["qno"], "type": r["qtype"],
|
||||
"stem": r["stem"], "opts": opts, "answer": r["answer"],
|
||||
"analysis": r["analysis"] or "", "note": r["note"] or "", "url": r["url"],
|
||||
})
|
||||
# 去重:同一 年份+题号+考点 只留一条(跨科目重复考点会重复出现)
|
||||
seen, uniq = set(), []
|
||||
for q in sorted(qs, key=lambda x: (x["year"], x["qno"])):
|
||||
key = (q["year"], q["qno"], q["kp"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
uniq.append(q)
|
||||
return uniq
|
||||
|
||||
|
||||
def fmt_options(opts):
|
||||
return " ".join(f"{o['letter']}. {o['text']}" for o in opts)
|
||||
|
||||
|
||||
def fmt_analysis(a, limit=240):
|
||||
a = re.sub(r"\n+", " ", a).strip()
|
||||
if len(a) > limit:
|
||||
return a[:limit] + "……"
|
||||
return a
|
||||
|
||||
|
||||
def build_section(qs):
|
||||
if not qs:
|
||||
return ""
|
||||
lines = [
|
||||
"### 真题精练(真题营 2010—2026 全量真题)",
|
||||
"",
|
||||
"> 来源:真题营 https://zhenti.zalize.com/zhenti/kaodian(2010—2026 历年真题,含答案与解析;已入库 SQLite 题库 `题库/zhenti_kaodian.db`,共 544 题)。以下收录本专题对应考点的全部历年真题,按年份排序,可直接闭卷刷。",
|
||||
"",
|
||||
]
|
||||
for i, q in enumerate(qs, 1):
|
||||
qtype = "多选" if q["type"] == "多选" else "单选"
|
||||
lines.append(f"**{i}. {q['year']}-{q['qno']} {qtype} · {q['kp']}**")
|
||||
lines.append(q["stem"])
|
||||
if q["opts"]:
|
||||
lines.append(fmt_options(q["opts"]))
|
||||
note = f" {q['note']}" if q["note"] else ""
|
||||
lines.append(f"**答案 {q['answer']}**{note}|**解析**:{fmt_analysis(q['analysis'])}")
|
||||
lines.append(f"> 原题出处:{q['url']}")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
qs = load_questions()
|
||||
by_topic = {}
|
||||
for q in qs:
|
||||
t = TOPIC_MAP.get(q["kp"])
|
||||
if t is None:
|
||||
print(f"!! 未映射考点: {q['kp']}", flush=True)
|
||||
continue
|
||||
by_topic.setdefault(t, []).append(q)
|
||||
|
||||
report = []
|
||||
total_chars = 0
|
||||
for t in range(1, 10):
|
||||
path = os.path.join(NOTE_DIR, f"专题{t} 某某.md")
|
||||
# 实际文件名按目录列出
|
||||
# 直接用 glob 找文件
|
||||
import glob
|
||||
files = {int(m.group(1)): m.group(0) for m in
|
||||
(re.match(r"专题(\d+) (.+?)\.md", os.path.basename(p)) for p in
|
||||
glob.glob(os.path.join(NOTE_DIR, "专题*.md"))) if m}
|
||||
for t in sorted(files):
|
||||
fname = files[t]
|
||||
path = os.path.join(NOTE_DIR, fname)
|
||||
text = open(path, encoding="utf-8").read()
|
||||
sec = build_section(by_topic.get(t, []))
|
||||
if not sec:
|
||||
report.append(f"{fname}: 无真题,跳过")
|
||||
continue
|
||||
anchor = "## 三、易混点速查"
|
||||
if anchor not in text:
|
||||
report.append(f"{fname}: 未找到锚点 {anchor},跳过")
|
||||
continue
|
||||
# 幂等: 移除已存在的旧真题精练区再重建
|
||||
m = re.search(r"### 真题精练.*?(?=\n## 三、)", text, re.S)
|
||||
if m:
|
||||
text = text[: m.start()] + text[m.end():]
|
||||
text = text.replace(anchor, sec + "\n\n" + anchor, 1)
|
||||
open(path, "w", encoding="utf-8").write(text)
|
||||
nq = len(by_topic.get(t, []))
|
||||
nchars = len(text)
|
||||
total_chars += nchars
|
||||
report.append(f"{fname}: 插入 {nq} 题,文件 {nchars} 字符")
|
||||
report.append(f"马原 9 专题共 {len(qs)} 真题,合计约 {total_chars} 字符")
|
||||
out = os.path.join(HERE, "_gen_report.txt")
|
||||
open(out, "w", encoding="utf-8").write("\n".join(report))
|
||||
print("\n".join(report), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
Reference in New Issue
Block a user