完形 4/5 篇重建,阅读专练五篇批改与五件套,作文素材库
- 完形专练:Text 4 重写、Text 5 换为闰秒/闰时文章,答案串去重后重新平衡 - 阅读专练:Text 1-5 批改记录 + 01-05 文章包(README/文章解读/词汇/背诵版/默写版) - 真题完形:2010 霍桑实验批改、背诵版、默写版 - 作文:可迁移句型素材库、大小作文范文、图表脚本与 PNG - 固定搭配:资源笔记、库设计、sqlite 构建脚本
This commit is contained in:
@@ -0,0 +1,367 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
构建《考研固定搭配库》:真题研究/固定搭配库.sqlite
|
||||
|
||||
- 词表来源:参考仓库/公共课/英语短语库/english-phrases/lists/npee.txt(考研 2936 条,CC BY-SA 4.0)
|
||||
- 语料来源:参考仓库/公共课/英语真题/{英语一,英语二}/*.pdf(1998-2026,按年份单卷,排除合订本)
|
||||
|
||||
可重复执行:每次运行会重建全部表。
|
||||
用法:python _build_collocation_db.py
|
||||
"""
|
||||
import io, os, re, sqlite3, datetime
|
||||
import pymupdf
|
||||
|
||||
ROOT = r'D:/考研/英语/真题研究'
|
||||
SRC = os.path.join(ROOT, r'参考仓库/公共课/英语短语库/english-phrases/lists/npee.txt')
|
||||
PDFROOT = os.path.join(ROOT, r'参考仓库/公共课/英语真题')
|
||||
DB = os.path.join(ROOT, '固定搭配库.sqlite')
|
||||
|
||||
NOW = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
REPO, URL, LIC = '2ndLA/english-phrases', 'https://github.com/2ndLA/english-phrases', 'CC BY-SA 4.0'
|
||||
|
||||
# ---------------------------------------------------------------- 词性 / 结构判定
|
||||
PREP = set('''of to with on in for at from into about as by upon over under against among between
|
||||
through without within toward towards than like beside beyond despite throughout per via amid onto
|
||||
since during until till except concerning regarding'''.split())
|
||||
|
||||
# 介词小词开头(介词框架)
|
||||
PREP_LEAD = PREP | set('''out up off before after because according regardless thanks owing prior
|
||||
instead apart aside due next close far ahead along'''.split())
|
||||
|
||||
# 明确非动词的常见首词(副词/形容词短语首词)
|
||||
NONVERB_HEAD = set('''ahead apart aside away back instead regardless owing according due thanks prior
|
||||
contrary alike akin aware devoid prone bound apt liable keen fond short void full free full'''.split())
|
||||
|
||||
# 副词性小品词(末位)
|
||||
PARTICLE_ADV = set('''up out off down away back through along apart aside forward forth together
|
||||
around round across over about aback'''.split())
|
||||
|
||||
ADJ_SUFFIX = ('able', 'ible', 'ous', 'ful', 'less', 'ive', 'ant', 'ent', 'al', 'ic', 'ary', 'ory')
|
||||
NOUN_SUFFIX = ('tion', 'sion', 'ment', 'ness', 'ity', 'ance', 'ence', 'ship', 'hood', 'ism', 'age')
|
||||
ADV_SUFFIX = ('ly',)
|
||||
|
||||
|
||||
def head_pos_guess(head, wc):
|
||||
"""首词词性粗判(启发式)"""
|
||||
if head == 'be':
|
||||
return 'be'
|
||||
if head in PREP:
|
||||
return '介词'
|
||||
if head in NONVERB_HEAD:
|
||||
return '形容词/副词'
|
||||
if head.endswith(ADV_SUFFIX) and len(head) > 4:
|
||||
return '副词'
|
||||
for suf in ADJ_SUFFIX:
|
||||
if head.endswith(suf) and len(head) > len(suf) + 1:
|
||||
return '形容词'
|
||||
for suf in NOUN_SUFFIX:
|
||||
if head.endswith(suf) and len(head) > len(suf) + 1:
|
||||
return '名词'
|
||||
if wc == 1:
|
||||
return '其他'
|
||||
return '动词'
|
||||
|
||||
|
||||
def tail_mark(tail):
|
||||
if tail in PARTICLE_ADV:
|
||||
return '小品词'
|
||||
if tail in PREP:
|
||||
return '介词'
|
||||
return '其他'
|
||||
|
||||
|
||||
def norm(phrase):
|
||||
s = phrase.lower().replace('’', "'")
|
||||
s = re.sub(r'\s+', ' ', s).strip()
|
||||
return s
|
||||
|
||||
|
||||
PLACE = [("sb.'s", '<ONESP>'), ("one's", '<ONESP>'), ('sb.', '<SB>'), ('sth.', '<STH>'), ('…', '<OBJ>')]
|
||||
|
||||
|
||||
def variant_key(phrase):
|
||||
"""去掉占位符与句首 be,用于把同一搭配的多种书写形式归到一组"""
|
||||
s = norm(phrase)
|
||||
for a, b in PLACE:
|
||||
s = s.replace(a, ' ')
|
||||
s = re.sub(r'^(be|to)\s+', '', s)
|
||||
s = re.sub(r'\s+', ' ', s).strip(' .')
|
||||
return s
|
||||
|
||||
|
||||
def placeholder_kind(phrase):
|
||||
kinds = []
|
||||
if '…' in phrase:
|
||||
kinds.append('…')
|
||||
if re.search(r"sb\.|sth\.", phrase):
|
||||
kinds.append('sb./sth.')
|
||||
if re.search(r"one's|sb\.'s", phrase):
|
||||
kinds.append("one's")
|
||||
return '+'.join(kinds) if kinds else None
|
||||
|
||||
|
||||
def classify(phrase):
|
||||
s = norm(phrase)
|
||||
toks = s.split(' ')
|
||||
wc = len(toks)
|
||||
head, tail = toks[0], toks[-1].strip('.,')
|
||||
hp = head_pos_guess(head, wc)
|
||||
tc = tail_mark(tail)
|
||||
if head == 'be':
|
||||
st = '系表结构'
|
||||
elif head in PREP_LEAD:
|
||||
st = '介词框架'
|
||||
elif tc == '小品词':
|
||||
st = '动/形+小品词'
|
||||
elif tc == '介词':
|
||||
st = ('动词' if hp in ('动词', '其他') else hp) + '+介词'
|
||||
elif wc == 1:
|
||||
st = '单词/复合词'
|
||||
else:
|
||||
st = '其他'
|
||||
return dict(normalized=s, variant_key=variant_key(phrase), head=head, head_pos_guess=hp,
|
||||
structure=st, tail=tail, tail_class=tc, word_count=wc,
|
||||
has_placeholder=1 if placeholder_kind(phrase) else 0,
|
||||
placeholder_kind=placeholder_kind(phrase))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 匹配模式
|
||||
def inflections(v):
|
||||
out = {v}
|
||||
if v.endswith('e'):
|
||||
out |= {v + 's', v + 'd', v[:-1] + 'ing'}
|
||||
elif v.endswith('y') and len(v) > 1 and v[-2] not in 'aeiou':
|
||||
out |= {v[:-1] + 'ies', v[:-1] + 'ied', v + 'ing'}
|
||||
else:
|
||||
out |= {v + 's', v + 'ed', v + 'ing'}
|
||||
if len(v) > 3 and v[-1] not in 'aeiouy' and v[-2] in 'aeiou' and v[-3] not in 'aeiou':
|
||||
out |= {v + v[-1] + 'ing', v + v[-1] + 'ed'}
|
||||
return sorted(out)
|
||||
|
||||
|
||||
OBJ = r"[a-z'\-]{1,18}(?:\s+[a-z'\-]{1,18}){0,2}"
|
||||
SUB = r"(?:sb\.|someone|somebody|us|him|her|them|me|you)"
|
||||
THG = r"(?:sth\.|something|it|them|that)"
|
||||
POSS = r"[a-z]+'s"
|
||||
|
||||
|
||||
def build_pattern(phrase, hp):
|
||||
s = norm(phrase)
|
||||
for a, b in PLACE:
|
||||
s = s.replace(a, b)
|
||||
esc = re.escape(s)
|
||||
toks = esc.split(' ')
|
||||
if toks and toks[0] == 'be':
|
||||
toks[0] = r'(?:be|is|are|was|were|been|being|am)'
|
||||
elif toks and re.fullmatch(r'[A-Za-z]+', toks[0]) and hp == '动词' and len(toks) > 1:
|
||||
toks[0] = '(?:' + '|'.join(inflections(toks[0])) + ')'
|
||||
toks = ['(?:do|does|did|done|doing)' if t == 'do' else t for t in toks]
|
||||
s = ' '.join(toks)
|
||||
s = s.replace('<ONESP>', POSS).replace('<SB>', SUB).replace('<STH>', THG).replace('<OBJ>', OBJ)
|
||||
return re.compile(r'\b' + s + r'\b', re.I)
|
||||
|
||||
|
||||
def sent_of(text, a, b):
|
||||
L = text.rfind('.', max(0, a - 250), a)
|
||||
R = text.find('.', b, b + 250)
|
||||
s = text[(L + 1 if L != -1 else max(0, a - 130)):(R + 1 if R != -1 else b + 130)]
|
||||
return re.sub(r'\s+', ' ', s).strip()[:320]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 语料
|
||||
def corpus_docs():
|
||||
out = []
|
||||
for sub in ('英语一', '英语二'):
|
||||
d = os.path.join(PDFROOT, sub)
|
||||
if not os.path.isdir(d):
|
||||
continue
|
||||
for fn in sorted(os.listdir(d)):
|
||||
if not fn.lower().endswith('.pdf') or '带书签' in fn or '无解析' in fn:
|
||||
continue
|
||||
m = re.search(r'(?:19|20)\d{2}', fn)
|
||||
if not m:
|
||||
continue
|
||||
out.append((int(m.group(0)), sub, fn, os.path.join(d, fn)))
|
||||
out.sort()
|
||||
return out
|
||||
|
||||
|
||||
def pdf_text(p):
|
||||
doc = pymupdf.open(p)
|
||||
t = ''.join(pg.get_text() for pg in doc)
|
||||
doc.close()
|
||||
t = t.replace('\u00ad', '').replace('\u2019', "'").replace('\u2018', "'").replace('\u2014', ' - ')
|
||||
return re.sub(r'\s+', ' ', t)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 建库
|
||||
def main():
|
||||
raw = [l.strip() for l in io.open(SRC, encoding='utf-8') if l.strip()]
|
||||
rows = []
|
||||
seen = set()
|
||||
for p in raw:
|
||||
if p in seen:
|
||||
continue
|
||||
seen.add(p)
|
||||
rows.append((p, classify(p)))
|
||||
print('词表:%d 条(去重后 %d)' % (len(raw), len(rows)))
|
||||
|
||||
docs = corpus_docs()
|
||||
print('语料:%d 份真题 PDF,抽取中…' % len(docs))
|
||||
texts = []
|
||||
for i, (y, t, fn, path) in enumerate(docs, 1):
|
||||
txt = pdf_text(path)
|
||||
texts.append((y, t, fn, txt))
|
||||
print(' [%2d/%d] %d %s %d 字符' % (i, len(docs), y, t, len(txt)), flush=True)
|
||||
|
||||
if os.path.exists(DB):
|
||||
os.remove(DB)
|
||||
con = sqlite3.connect(DB)
|
||||
con.executescript('''
|
||||
PRAGMA foreign_keys=ON;
|
||||
CREATE TABLE sources(
|
||||
source_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
list_name TEXT NOT NULL, repo TEXT NOT NULL, repo_url TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL, license TEXT NOT NULL, license_note TEXT,
|
||||
entry_count INTEGER NOT NULL, retrieved_at TEXT NOT NULL,
|
||||
UNIQUE(list_name, repo, file_path));
|
||||
CREATE TABLE phrases(
|
||||
phrase_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
phrase TEXT NOT NULL,
|
||||
normalized TEXT NOT NULL,
|
||||
variant_key TEXT NOT NULL,
|
||||
head TEXT, head_pos_guess TEXT,
|
||||
structure TEXT, tail TEXT, tail_class TEXT,
|
||||
word_count INTEGER NOT NULL,
|
||||
has_placeholder INTEGER NOT NULL, placeholder_kind TEXT,
|
||||
source_id INTEGER NOT NULL REFERENCES sources(source_id),
|
||||
list_name TEXT NOT NULL,
|
||||
variant_count INTEGER DEFAULT 1,
|
||||
exam_hits INTEGER DEFAULT 0,
|
||||
exam_docs TEXT, first_year INTEGER,
|
||||
tags TEXT,
|
||||
cn_meaning TEXT, usage_note TEXT, importance INTEGER,
|
||||
review_status TEXT DEFAULT '未复习',
|
||||
created_at TEXT NOT NULL,
|
||||
UNIQUE(phrase, list_name));
|
||||
CREATE TABLE corpus_docs(
|
||||
doc_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
exam_year INTEGER NOT NULL, exam_type TEXT NOT NULL,
|
||||
source_file TEXT NOT NULL, word_count INTEGER, char_count INTEGER, text TEXT,
|
||||
UNIQUE(exam_year, exam_type));
|
||||
CREATE TABLE occurrences(
|
||||
occ_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
phrase_id INTEGER NOT NULL REFERENCES phrases(phrase_id),
|
||||
doc_id INTEGER NOT NULL REFERENCES corpus_docs(doc_id),
|
||||
matched_text TEXT, sentence TEXT);
|
||||
CREATE INDEX idx_ph_head ON phrases(head);
|
||||
CREATE INDEX idx_ph_variant ON phrases(variant_key);
|
||||
CREATE INDEX idx_ph_structure ON phrases(structure);
|
||||
CREATE INDEX idx_ph_exam ON phrases(exam_hits);
|
||||
CREATE INDEX idx_occ_phrase ON occurrences(phrase_id);
|
||||
CREATE INDEX idx_occ_doc ON occurrences(doc_id);
|
||||
''')
|
||||
|
||||
cur = con.cursor()
|
||||
cur.execute('''INSERT INTO sources(list_name,repo,repo_url,file_path,license,license_note,entry_count,retrieved_at)
|
||||
VALUES(?,?,?,?,?,?,?,?)''',
|
||||
('考研', REPO, URL, 'lists/npee.txt', LIC,
|
||||
'fork of moodHappy/english-phrases(上游考研表仅 875 条)', len(rows), NOW))
|
||||
sid = cur.lastrowid
|
||||
|
||||
for y, t, fn, txt in texts:
|
||||
cur.execute('''INSERT INTO corpus_docs(exam_year,exam_type,source_file,word_count,char_count,text)
|
||||
VALUES(?,?,?,?,?,?)''', (y, t, fn, len(txt.split()), len(txt), txt))
|
||||
doc_ids = [r[0] for r in cur.execute('SELECT doc_id FROM corpus_docs ORDER BY doc_id')]
|
||||
|
||||
print('逐条检索中(%d 条 × %d 卷)…' % (len(rows), len(texts)))
|
||||
hit_phrases = 0
|
||||
total_occ = 0
|
||||
for phrase, info in rows:
|
||||
cs = classify(phrase) # 与 rows 中一致,取词性供模式构建
|
||||
pat = build_pattern(phrase, cs['head_pos_guess'])
|
||||
info = dict(info)
|
||||
cur.execute('''INSERT INTO phrases(phrase,normalized,variant_key,head,head_pos_guess,structure,tail,
|
||||
tail_class,word_count,has_placeholder,placeholder_kind,source_id,list_name,created_at)
|
||||
VALUES(:phrase,:normalized,:variant_key,:head,:head_pos_guess,:structure,:tail,
|
||||
:tail_class,:word_count,:has_placeholder,:placeholder_kind,:sid,:ln,:now)''',
|
||||
dict(info, phrase=phrase, sid=sid, ln='考研', now=NOW))
|
||||
pid = cur.lastrowid
|
||||
hits, dlabels, years = 0, set(), []
|
||||
for (y, t, fn, txt), did in zip(texts, doc_ids):
|
||||
n = 0
|
||||
for m in pat.finditer(txt):
|
||||
n += 1
|
||||
total_occ += 1
|
||||
if hits < 3:
|
||||
cur.execute('INSERT INTO occurrences(phrase_id,doc_id,matched_text,sentence) VALUES(?,?,?,?)',
|
||||
(pid, did, m.group(0)[:60], sent_of(txt, m.start(), m.end())))
|
||||
hits += 1
|
||||
if n >= 8:
|
||||
break
|
||||
if n:
|
||||
dlabels.add('%d%s' % (y, '英一' if t == '英语一' else '英二'))
|
||||
years.append(y)
|
||||
if hits:
|
||||
hit_phrases += 1
|
||||
cur.execute('UPDATE phrases SET exam_hits=?, exam_docs=?, first_year=? WHERE phrase_id=?',
|
||||
(hits, ','.join(sorted(dlabels)), min(years), pid))
|
||||
|
||||
# 变体计数
|
||||
cur.execute('''UPDATE phrases SET variant_count=(SELECT COUNT(*) FROM phrases p2
|
||||
WHERE p2.variant_key=phrases.variant_key)''')
|
||||
|
||||
# 标签种子:2010 真题完形做错的 4 个搭配
|
||||
WRONG2010 = {'be subjected to': '2010完形错题', 'subject to': '2010完形错题',
|
||||
'end up': '2010完形错题', 'end up doing': '2010完形错题',
|
||||
'so long as': '2010完形错题', 'as long as': '2010完形错题',
|
||||
'contrary to': '2010完形错题', 'on the contrary': '2010完形错题'}
|
||||
for k, tag in WRONG2010.items():
|
||||
cur.execute('UPDATE phrases SET tags=? WHERE variant_key=?', (tag, k))
|
||||
|
||||
# 视图
|
||||
con.executescript('''
|
||||
CREATE VIEW v_particle_stats AS
|
||||
SELECT tail_class, tail, COUNT(*) AS n FROM phrases GROUP BY tail_class, tail ORDER BY n DESC;
|
||||
CREATE VIEW v_structure_stats AS
|
||||
SELECT structure, COUNT(*) AS n FROM phrases GROUP BY structure ORDER BY n DESC;
|
||||
CREATE VIEW v_variants AS
|
||||
SELECT variant_key, COUNT(*) AS n, GROUP_CONCAT(phrase, ' | ') AS forms
|
||||
FROM phrases GROUP BY variant_key HAVING n>1 ORDER BY n DESC;
|
||||
CREATE VIEW v_exam_phrases AS
|
||||
SELECT phrase, structure, exam_hits, first_year, exam_docs FROM phrases
|
||||
WHERE exam_hits>0 ORDER BY exam_hits DESC, phrase;
|
||||
CREATE VIEW v_todo_cn AS
|
||||
SELECT phrase_id, phrase, structure FROM phrases WHERE cn_meaning IS NULL ORDER BY phrase;
|
||||
''')
|
||||
con.commit()
|
||||
|
||||
# 汇总
|
||||
n_ph = cur.execute('SELECT COUNT(*) FROM phrases').fetchone()[0]
|
||||
n_doc = cur.execute('SELECT COUNT(*) FROM corpus_docs').fetchone()[0]
|
||||
n_occ = cur.execute('SELECT COUNT(*) FROM occurrences').fetchone()[0]
|
||||
print()
|
||||
print('=' * 66)
|
||||
print('库文件 : %s (%.1f KB)' % (DB, os.path.getsize(DB) / 1024))
|
||||
print('phrases : %d 条' % n_ph)
|
||||
print('corpus_docs : %d 份真题' % n_doc)
|
||||
print('occurrences : %d 条命中样本' % n_occ)
|
||||
print('真题命中搭配: %d 条(占 %.1f%%)' % (hit_phrases, 100.0 * hit_phrases / n_ph))
|
||||
print('命中总次数 : %d' % total_occ)
|
||||
print('-' * 66)
|
||||
print('结构分布:')
|
||||
for s, n in cur.execute('SELECT structure, COUNT(*) FROM phrases GROUP BY structure ORDER BY 2 DESC'):
|
||||
print(' %-16s %5d (%.1f%%)' % (s, n, 100.0 * n / n_ph))
|
||||
print('-' * 66)
|
||||
print('真题命中 Top 15:')
|
||||
for r in cur.execute('SELECT phrase, exam_hits, exam_docs FROM v_exam_phrases LIMIT 15'):
|
||||
print(' %-32s %3d 次 %s' % (r[0], r[1], (r[2] or '')[:48]))
|
||||
print('-' * 66)
|
||||
print('变体组最多的 10 组:')
|
||||
for r in cur.execute('SELECT variant_key, n FROM v_variants LIMIT 10'):
|
||||
print(' %-30s %d 种形式' % (r[0], r[1]))
|
||||
con.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user