Files

130 lines
4.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""真题营·考研政治历年分析题爬虫 → SQLite(表 fenxiti)
来源: https://zhenti.zalize.com/zhenti/fenxiti (2010-2026 分析题 85 道,材料为概述、设问为真题原文、参考答案要点免费预览1/4)
礼貌爬取: 0.5~0.8s 随机间隔; 指数退避重试; 断点续爬。
用法: python crawl_fenxiti.py
"""
import os, re, time, random, sqlite3, sys
import requests
from bs4 import BeautifulSoup
BASE = "https://zhenti.zalize.com"
INDEX = BASE + "/zhenti/fenxiti"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36"}
HERE = os.path.dirname(os.path.abspath(__file__))
DB = os.path.join(HERE, "zhenti_kaodian.db")
def get(url, retries=3):
for i in range(retries):
try:
r = requests.get(url, headers=HEADERS, timeout=30)
if r.status_code == 200:
return r.text
print(f" HTTP {r.status_code} {url}", flush=True)
except Exception as e:
print(f" ERR {type(e).__name__} {url}", flush=True)
time.sleep(2 ** i)
return None
def parse_index(html):
soup = BeautifulSoup(html, "html.parser")
urls = []
for a in soup.find_all("a", href=re.compile(r"^/zhenti/fenxiti/\d{4}-\d{2}$")):
urls.append(BASE + a["href"])
# 去重保序
seen = set()
out = []
for u in urls:
if u not in seen:
seen.add(u)
out.append(u)
return out
def parse_page(html):
soup = BeautifulSoup(html, "html.parser")
h1 = soup.find("h1")
m = re.search(r"(\d{4}) 年考研政治分析题第 (\d+) 题", h1.get_text(strip=True) if h1 else "")
if not m:
return None
year, qno = int(m.group(1)), int(m.group(2))
# 科目与考点: p 文本形如 "马原·哲学 · <a>主观能动性与客观规律性</a> · 材料为原创概述..."
subj = kp = ""
p = soup.find("p", class_=re.compile(r"^mt-2 text-sm text-slate-600"))
if p:
txt = p.get_text(" ", strip=True)
subj = txt.split(" · ")[0].strip() if " · " in txt else ""
kp_a = p.find("a")
if kp_a:
kp = kp_a.get_text(strip=True)
article = soup.find("article")
stem = article.get_text("\n", strip=True) if article else ""
ans = ""
box = soup.find("div", class_=re.compile(r"border-brand-200"))
if box:
pts = box.find_all("p", class_="mt-2")
parts = []
for pt in pts:
t = pt.get_text(" ", strip=True)
if t and "全部要点已整理好" not in t and "注册后可免费" not in t:
parts.append(t)
ans = "\n".join(parts)
return {"year": year, "qno": qno, "subject": subj, "kaodian": kp,
"stem": stem, "answer_preview": ans, "url": f"{BASE}/zhenti/fenxiti/{year}-{qno}"}
def main():
con = sqlite3.connect(DB)
con.executescript("""
CREATE TABLE IF NOT EXISTS fenxiti(
id INTEGER PRIMARY KEY AUTOINCREMENT,
year INTEGER, qno INTEGER,
subject TEXT, kaodian TEXT,
stem TEXT, answer_preview TEXT, url TEXT,
UNIQUE(year, qno)
);
""")
con.commit()
cur = con.cursor()
idx = get(INDEX)
if not idx:
print("索引页失败", flush=True)
return 1
urls = parse_index(idx)
print(f"分析题索引: {len(urls)} 道", flush=True)
cur.execute("SELECT year, qno, subject, kaodian FROM fenxiti")
done = {(r[0], r[1]): (r[2], r[3]) for r in cur.fetchall()}
bad_kp = {"哲学", "毛中特", "史纲", "思修法基", "形势与政策", "马原"}
for i, u in enumerate(urls, 1):
m = re.search(r"/(\d{4})-(\d{2})$", u)
key = (int(m.group(1)), int(m.group(2)))
if key in done and done[key][0] and done[key][1] and done[key][1] not in bad_kp:
print(f"[{i}/{len(urls)}] 跳过: {key}", flush=True)
continue
time.sleep(0.5 + random.random() * 0.3)
html = get(u)
if not html:
print(f"[{i}/{len(urls)}] 失败: {u}", flush=True)
continue
q = parse_page(html)
if not q:
print(f"[{i}/{len(urls)}] 解析失败: {u}", flush=True)
continue
cur.execute("INSERT OR REPLACE INTO fenxiti(year,qno,subject,kaodian,stem,answer_preview,url) "
"VALUES(?,?,?,?,?,?,?)",
(q["year"], q["qno"], q["subject"], q["kaodian"], q["stem"], q["answer_preview"], q["url"]))
con.commit()
print(f"[{i}/{len(urls)}] ✓ {q['year']}-{q['qno']} {q['subject']} | {q['kaodian']}", flush=True)
cur.execute("SELECT COUNT(*) FROM fenxiti")
print(f"完成: fenxiti 表共 {cur.fetchone()[0]} 道", flush=True)
con.close()
return 0
if __name__ == "__main__":
sys.exit(main())