完形 4/5 篇重建,阅读专练五篇批改与五件套,作文素材库

- 完形专练:Text 4 重写、Text 5 换为闰秒/闰时文章,答案串去重后重新平衡
- 阅读专练:Text 1-5 批改记录 + 01-05 文章包(README/文章解读/词汇/背诵版/默写版)
- 真题完形:2010 霍桑实验批改、背诵版、默写版
- 作文:可迁移句型素材库、大小作文范文、图表脚本与 PNG
- 固定搭配:资源笔记、库设计、sqlite 构建脚本
This commit is contained in:
2026-10-20 22:11:58 +08:00
parent 58d58cc60b
commit 54e4ecae16
93 changed files with 12880 additions and 675 deletions
+367
View File
@@ -0,0 +1,367 @@
# -*- coding: utf-8 -*-
"""
构建《考研固定搭配库》:真题研究/固定搭配库.sqlite
- 词表来源:参考仓库/公共课/英语短语库/english-phrases/lists/npee.txt(考研 2936 条,CC BY-SA 4.0)
- 语料来源:参考仓库/公共课/英语真题/{英语一,英语二}/*.pdf(1998-2026,按年份单卷,排除合订本)
可重复执行:每次运行会重建全部表。
用法:python _build_collocation_db.py
"""
import io, os, re, sqlite3, datetime
import pymupdf
ROOT = r'D:/考研/英语/真题研究'
SRC = os.path.join(ROOT, r'参考仓库/公共课/英语短语库/english-phrases/lists/npee.txt')
PDFROOT = os.path.join(ROOT, r'参考仓库/公共课/英语真题')
DB = os.path.join(ROOT, '固定搭配库.sqlite')
NOW = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
REPO, URL, LIC = '2ndLA/english-phrases', 'https://github.com/2ndLA/english-phrases', 'CC BY-SA 4.0'
# ---------------------------------------------------------------- 词性 / 结构判定
PREP = set('''of to with on in for at from into about as by upon over under against among between
through without within toward towards than like beside beyond despite throughout per via amid onto
since during until till except concerning regarding'''.split())
# 介词小词开头(介词框架)
PREP_LEAD = PREP | set('''out up off before after because according regardless thanks owing prior
instead apart aside due next close far ahead along'''.split())
# 明确非动词的常见首词(副词/形容词短语首词)
NONVERB_HEAD = set('''ahead apart aside away back instead regardless owing according due thanks prior
contrary alike akin aware devoid prone bound apt liable keen fond short void full free full'''.split())
# 副词性小品词(末位)
PARTICLE_ADV = set('''up out off down away back through along apart aside forward forth together
around round across over about aback'''.split())
ADJ_SUFFIX = ('able', 'ible', 'ous', 'ful', 'less', 'ive', 'ant', 'ent', 'al', 'ic', 'ary', 'ory')
NOUN_SUFFIX = ('tion', 'sion', 'ment', 'ness', 'ity', 'ance', 'ence', 'ship', 'hood', 'ism', 'age')
ADV_SUFFIX = ('ly',)
def head_pos_guess(head, wc):
"""首词词性粗判(启发式)"""
if head == 'be':
return 'be'
if head in PREP:
return '介词'
if head in NONVERB_HEAD:
return '形容词/副词'
if head.endswith(ADV_SUFFIX) and len(head) > 4:
return '副词'
for suf in ADJ_SUFFIX:
if head.endswith(suf) and len(head) > len(suf) + 1:
return '形容词'
for suf in NOUN_SUFFIX:
if head.endswith(suf) and len(head) > len(suf) + 1:
return '名词'
if wc == 1:
return '其他'
return '动词'
def tail_mark(tail):
if tail in PARTICLE_ADV:
return '小品词'
if tail in PREP:
return '介词'
return '其他'
def norm(phrase):
s = phrase.lower().replace('’', "'")
s = re.sub(r'\s+', ' ', s).strip()
return s
PLACE = [("sb.'s", '<ONESP>'), ("one's", '<ONESP>'), ('sb.', '<SB>'), ('sth.', '<STH>'), ('…', '<OBJ>')]
def variant_key(phrase):
"""去掉占位符与句首 be,用于把同一搭配的多种书写形式归到一组"""
s = norm(phrase)
for a, b in PLACE:
s = s.replace(a, ' ')
s = re.sub(r'^(be|to)\s+', '', s)
s = re.sub(r'\s+', ' ', s).strip(' .')
return s
def placeholder_kind(phrase):
kinds = []
if '…' in phrase:
kinds.append('…')
if re.search(r"sb\.|sth\.", phrase):
kinds.append('sb./sth.')
if re.search(r"one's|sb\.'s", phrase):
kinds.append("one's")
return '+'.join(kinds) if kinds else None
def classify(phrase):
s = norm(phrase)
toks = s.split(' ')
wc = len(toks)
head, tail = toks[0], toks[-1].strip('.,')
hp = head_pos_guess(head, wc)
tc = tail_mark(tail)
if head == 'be':
st = '系表结构'
elif head in PREP_LEAD:
st = '介词框架'
elif tc == '小品词':
st = '动/形+小品词'
elif tc == '介词':
st = ('动词' if hp in ('动词', '其他') else hp) + '+介词'
elif wc == 1:
st = '单词/复合词'
else:
st = '其他'
return dict(normalized=s, variant_key=variant_key(phrase), head=head, head_pos_guess=hp,
structure=st, tail=tail, tail_class=tc, word_count=wc,
has_placeholder=1 if placeholder_kind(phrase) else 0,
placeholder_kind=placeholder_kind(phrase))
# ---------------------------------------------------------------- 匹配模式
def inflections(v):
out = {v}
if v.endswith('e'):
out |= {v + 's', v + 'd', v[:-1] + 'ing'}
elif v.endswith('y') and len(v) > 1 and v[-2] not in 'aeiou':
out |= {v[:-1] + 'ies', v[:-1] + 'ied', v + 'ing'}
else:
out |= {v + 's', v + 'ed', v + 'ing'}
if len(v) > 3 and v[-1] not in 'aeiouy' and v[-2] in 'aeiou' and v[-3] not in 'aeiou':
out |= {v + v[-1] + 'ing', v + v[-1] + 'ed'}
return sorted(out)
OBJ = r"[a-z'\-]{1,18}(?:\s+[a-z'\-]{1,18}){0,2}"
SUB = r"(?:sb\.|someone|somebody|us|him|her|them|me|you)"
THG = r"(?:sth\.|something|it|them|that)"
POSS = r"[a-z]+'s"
def build_pattern(phrase, hp):
s = norm(phrase)
for a, b in PLACE:
s = s.replace(a, b)
esc = re.escape(s)
toks = esc.split(' ')
if toks and toks[0] == 'be':
toks[0] = r'(?:be|is|are|was|were|been|being|am)'
elif toks and re.fullmatch(r'[A-Za-z]+', toks[0]) and hp == '动词' and len(toks) > 1:
toks[0] = '(?:' + '|'.join(inflections(toks[0])) + ')'
toks = ['(?:do|does|did|done|doing)' if t == 'do' else t for t in toks]
s = ' '.join(toks)
s = s.replace('<ONESP>', POSS).replace('<SB>', SUB).replace('<STH>', THG).replace('<OBJ>', OBJ)
return re.compile(r'\b' + s + r'\b', re.I)
def sent_of(text, a, b):
L = text.rfind('.', max(0, a - 250), a)
R = text.find('.', b, b + 250)
s = text[(L + 1 if L != -1 else max(0, a - 130)):(R + 1 if R != -1 else b + 130)]
return re.sub(r'\s+', ' ', s).strip()[:320]
# ---------------------------------------------------------------- 语料
def corpus_docs():
out = []
for sub in ('英语一', '英语二'):
d = os.path.join(PDFROOT, sub)
if not os.path.isdir(d):
continue
for fn in sorted(os.listdir(d)):
if not fn.lower().endswith('.pdf') or '带书签' in fn or '无解析' in fn:
continue
m = re.search(r'(?:19|20)\d{2}', fn)
if not m:
continue
out.append((int(m.group(0)), sub, fn, os.path.join(d, fn)))
out.sort()
return out
def pdf_text(p):
doc = pymupdf.open(p)
t = ''.join(pg.get_text() for pg in doc)
doc.close()
t = t.replace('\u00ad', '').replace('\u2019', "'").replace('\u2018', "'").replace('\u2014', ' - ')
return re.sub(r'\s+', ' ', t)
# ---------------------------------------------------------------- 建库
def main():
raw = [l.strip() for l in io.open(SRC, encoding='utf-8') if l.strip()]
rows = []
seen = set()
for p in raw:
if p in seen:
continue
seen.add(p)
rows.append((p, classify(p)))
print('词表:%d 条(去重后 %d)' % (len(raw), len(rows)))
docs = corpus_docs()
print('语料:%d 份真题 PDF,抽取中…' % len(docs))
texts = []
for i, (y, t, fn, path) in enumerate(docs, 1):
txt = pdf_text(path)
texts.append((y, t, fn, txt))
print(' [%2d/%d] %d %s %d 字符' % (i, len(docs), y, t, len(txt)), flush=True)
if os.path.exists(DB):
os.remove(DB)
con = sqlite3.connect(DB)
con.executescript('''
PRAGMA foreign_keys=ON;
CREATE TABLE sources(
source_id INTEGER PRIMARY KEY AUTOINCREMENT,
list_name TEXT NOT NULL, repo TEXT NOT NULL, repo_url TEXT NOT NULL,
file_path TEXT NOT NULL, license TEXT NOT NULL, license_note TEXT,
entry_count INTEGER NOT NULL, retrieved_at TEXT NOT NULL,
UNIQUE(list_name, repo, file_path));
CREATE TABLE phrases(
phrase_id INTEGER PRIMARY KEY AUTOINCREMENT,
phrase TEXT NOT NULL,
normalized TEXT NOT NULL,
variant_key TEXT NOT NULL,
head TEXT, head_pos_guess TEXT,
structure TEXT, tail TEXT, tail_class TEXT,
word_count INTEGER NOT NULL,
has_placeholder INTEGER NOT NULL, placeholder_kind TEXT,
source_id INTEGER NOT NULL REFERENCES sources(source_id),
list_name TEXT NOT NULL,
variant_count INTEGER DEFAULT 1,
exam_hits INTEGER DEFAULT 0,
exam_docs TEXT, first_year INTEGER,
tags TEXT,
cn_meaning TEXT, usage_note TEXT, importance INTEGER,
review_status TEXT DEFAULT '未复习',
created_at TEXT NOT NULL,
UNIQUE(phrase, list_name));
CREATE TABLE corpus_docs(
doc_id INTEGER PRIMARY KEY AUTOINCREMENT,
exam_year INTEGER NOT NULL, exam_type TEXT NOT NULL,
source_file TEXT NOT NULL, word_count INTEGER, char_count INTEGER, text TEXT,
UNIQUE(exam_year, exam_type));
CREATE TABLE occurrences(
occ_id INTEGER PRIMARY KEY AUTOINCREMENT,
phrase_id INTEGER NOT NULL REFERENCES phrases(phrase_id),
doc_id INTEGER NOT NULL REFERENCES corpus_docs(doc_id),
matched_text TEXT, sentence TEXT);
CREATE INDEX idx_ph_head ON phrases(head);
CREATE INDEX idx_ph_variant ON phrases(variant_key);
CREATE INDEX idx_ph_structure ON phrases(structure);
CREATE INDEX idx_ph_exam ON phrases(exam_hits);
CREATE INDEX idx_occ_phrase ON occurrences(phrase_id);
CREATE INDEX idx_occ_doc ON occurrences(doc_id);
''')
cur = con.cursor()
cur.execute('''INSERT INTO sources(list_name,repo,repo_url,file_path,license,license_note,entry_count,retrieved_at)
VALUES(?,?,?,?,?,?,?,?)''',
('考研', REPO, URL, 'lists/npee.txt', LIC,
'fork of moodHappy/english-phrases(上游考研表仅 875 条)', len(rows), NOW))
sid = cur.lastrowid
for y, t, fn, txt in texts:
cur.execute('''INSERT INTO corpus_docs(exam_year,exam_type,source_file,word_count,char_count,text)
VALUES(?,?,?,?,?,?)''', (y, t, fn, len(txt.split()), len(txt), txt))
doc_ids = [r[0] for r in cur.execute('SELECT doc_id FROM corpus_docs ORDER BY doc_id')]
print('逐条检索中(%d 条 × %d 卷)…' % (len(rows), len(texts)))
hit_phrases = 0
total_occ = 0
for phrase, info in rows:
cs = classify(phrase) # 与 rows 中一致,取词性供模式构建
pat = build_pattern(phrase, cs['head_pos_guess'])
info = dict(info)
cur.execute('''INSERT INTO phrases(phrase,normalized,variant_key,head,head_pos_guess,structure,tail,
tail_class,word_count,has_placeholder,placeholder_kind,source_id,list_name,created_at)
VALUES(:phrase,:normalized,:variant_key,:head,:head_pos_guess,:structure,:tail,
:tail_class,:word_count,:has_placeholder,:placeholder_kind,:sid,:ln,:now)''',
dict(info, phrase=phrase, sid=sid, ln='考研', now=NOW))
pid = cur.lastrowid
hits, dlabels, years = 0, set(), []
for (y, t, fn, txt), did in zip(texts, doc_ids):
n = 0
for m in pat.finditer(txt):
n += 1
total_occ += 1
if hits < 3:
cur.execute('INSERT INTO occurrences(phrase_id,doc_id,matched_text,sentence) VALUES(?,?,?,?)',
(pid, did, m.group(0)[:60], sent_of(txt, m.start(), m.end())))
hits += 1
if n >= 8:
break
if n:
dlabels.add('%d%s' % (y, '英一' if t == '英语一' else '英二'))
years.append(y)
if hits:
hit_phrases += 1
cur.execute('UPDATE phrases SET exam_hits=?, exam_docs=?, first_year=? WHERE phrase_id=?',
(hits, ','.join(sorted(dlabels)), min(years), pid))
# 变体计数
cur.execute('''UPDATE phrases SET variant_count=(SELECT COUNT(*) FROM phrases p2
WHERE p2.variant_key=phrases.variant_key)''')
# 标签种子:2010 真题完形做错的 4 个搭配
WRONG2010 = {'be subjected to': '2010完形错题', 'subject to': '2010完形错题',
'end up': '2010完形错题', 'end up doing': '2010完形错题',
'so long as': '2010完形错题', 'as long as': '2010完形错题',
'contrary to': '2010完形错题', 'on the contrary': '2010完形错题'}
for k, tag in WRONG2010.items():
cur.execute('UPDATE phrases SET tags=? WHERE variant_key=?', (tag, k))
# 视图
con.executescript('''
CREATE VIEW v_particle_stats AS
SELECT tail_class, tail, COUNT(*) AS n FROM phrases GROUP BY tail_class, tail ORDER BY n DESC;
CREATE VIEW v_structure_stats AS
SELECT structure, COUNT(*) AS n FROM phrases GROUP BY structure ORDER BY n DESC;
CREATE VIEW v_variants AS
SELECT variant_key, COUNT(*) AS n, GROUP_CONCAT(phrase, ' | ') AS forms
FROM phrases GROUP BY variant_key HAVING n>1 ORDER BY n DESC;
CREATE VIEW v_exam_phrases AS
SELECT phrase, structure, exam_hits, first_year, exam_docs FROM phrases
WHERE exam_hits>0 ORDER BY exam_hits DESC, phrase;
CREATE VIEW v_todo_cn AS
SELECT phrase_id, phrase, structure FROM phrases WHERE cn_meaning IS NULL ORDER BY phrase;
''')
con.commit()
# 汇总
n_ph = cur.execute('SELECT COUNT(*) FROM phrases').fetchone()[0]
n_doc = cur.execute('SELECT COUNT(*) FROM corpus_docs').fetchone()[0]
n_occ = cur.execute('SELECT COUNT(*) FROM occurrences').fetchone()[0]
print()
print('=' * 66)
print('库文件 : %s (%.1f KB)' % (DB, os.path.getsize(DB) / 1024))
print('phrases : %d 条' % n_ph)
print('corpus_docs : %d 份真题' % n_doc)
print('occurrences : %d 条命中样本' % n_occ)
print('真题命中搭配: %d 条(占 %.1f%%)' % (hit_phrases, 100.0 * hit_phrases / n_ph))
print('命中总次数 : %d' % total_occ)
print('-' * 66)
print('结构分布:')
for s, n in cur.execute('SELECT structure, COUNT(*) FROM phrases GROUP BY structure ORDER BY 2 DESC'):
print(' %-16s %5d (%.1f%%)' % (s, n, 100.0 * n / n_ph))
print('-' * 66)
print('真题命中 Top 15:')
for r in cur.execute('SELECT phrase, exam_hits, exam_docs FROM v_exam_phrases LIMIT 15'):
print(' %-32s %3d 次 %s' % (r[0], r[1], (r[2] or '')[:48]))
print('-' * 66)
print('变体组最多的 10 组:')
for r in cur.execute('SELECT variant_key, n FROM v_variants LIMIT 10'):
print(' %-30s %d 种形式' % (r[0], r[1]))
con.close()
if __name__ == '__main__':
main()