368 lines
15 KiB
Python
368 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
||||
|
|
"""
|
|||
|
|
构建《考研固定搭配库》:真题研究/固定搭配库.sqlite
|
|||
|
|
|
|||
|
|
- 词表来源:参考仓库/公共课/英语短语库/english-phrases/lists/npee.txt(考研 2936 条,CC BY-SA 4.0)
|
|||
|
|
- 语料来源:参考仓库/公共课/英语真题/{英语一,英语二}/*.pdf(1998-2026,按年份单卷,排除合订本)
|
|||
|
|
|
|||
|
|
可重复执行:每次运行会重建全部表。
|
|||
|
|
用法:python _build_collocation_db.py
|
|||
|
|
"""
|
|||
|
|
import io, os, re, sqlite3, datetime
|
|||
|
|
import pymupdf
|
|||
|
|
|
|||
|
|
ROOT = r'D:/考研/英语/真题研究'
|
|||
|
|
SRC = os.path.join(ROOT, r'参考仓库/公共课/英语短语库/english-phrases/lists/npee.txt')
|
|||
|
|
PDFROOT = os.path.join(ROOT, r'参考仓库/公共课/英语真题')
|
|||
|
|
DB = os.path.join(ROOT, '固定搭配库.sqlite')
|
|||
|
|
|
|||
|
|
NOW = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|||
|
|
REPO, URL, LIC = '2ndLA/english-phrases', 'https://github.com/2ndLA/english-phrases', 'CC BY-SA 4.0'
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 词性 / 结构判定
|
|||
|
|
PREP = set('''of to with on in for at from into about as by upon over under against among between
|
|||
|
|
through without within toward towards than like beside beyond despite throughout per via amid onto
|
|||
|
|
since during until till except concerning regarding'''.split())
|
|||
|
|
|
|||
|
|
# 介词小词开头(介词框架)
|
|||
|
|
PREP_LEAD = PREP | set('''out up off before after because according regardless thanks owing prior
|
|||
|
|
instead apart aside due next close far ahead along'''.split())
|
|||
|
|
|
|||
|
|
# 明确非动词的常见首词(副词/形容词短语首词)
|
|||
|
|
NONVERB_HEAD = set('''ahead apart aside away back instead regardless owing according due thanks prior
|
|||
|
|
contrary alike akin aware devoid prone bound apt liable keen fond short void full free full'''.split())
|
|||
|
|
|
|||
|
|
# 副词性小品词(末位)
|
|||
|
|
PARTICLE_ADV = set('''up out off down away back through along apart aside forward forth together
|
|||
|
|
around round across over about aback'''.split())
|
|||
|
|
|
|||
|
|
ADJ_SUFFIX = ('able', 'ible', 'ous', 'ful', 'less', 'ive', 'ant', 'ent', 'al', 'ic', 'ary', 'ory')
|
|||
|
|
NOUN_SUFFIX = ('tion', 'sion', 'ment', 'ness', 'ity', 'ance', 'ence', 'ship', 'hood', 'ism', 'age')
|
|||
|
|
ADV_SUFFIX = ('ly',)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def head_pos_guess(head, wc):
|
|||
|
|
"""首词词性粗判(启发式)"""
|
|||
|
|
if head == 'be':
|
|||
|
|
return 'be'
|
|||
|
|
if head in PREP:
|
|||
|
|
return '介词'
|
|||
|
|
if head in NONVERB_HEAD:
|
|||
|
|
return '形容词/副词'
|
|||
|
|
if head.endswith(ADV_SUFFIX) and len(head) > 4:
|
|||
|
|
return '副词'
|
|||
|
|
for suf in ADJ_SUFFIX:
|
|||
|
|
if head.endswith(suf) and len(head) > len(suf) + 1:
|
|||
|
|
return '形容词'
|
|||
|
|
for suf in NOUN_SUFFIX:
|
|||
|
|
if head.endswith(suf) and len(head) > len(suf) + 1:
|
|||
|
|
return '名词'
|
|||
|
|
if wc == 1:
|
|||
|
|
return '其他'
|
|||
|
|
return '动词'
|
|||
|
|
|
|||
|
|
|
|||
|
|
def tail_mark(tail):
|
|||
|
|
if tail in PARTICLE_ADV:
|
|||
|
|
return '小品词'
|
|||
|
|
if tail in PREP:
|
|||
|
|
return '介词'
|
|||
|
|
return '其他'
|
|||
|
|
|
|||
|
|
|
|||
|
|
def norm(phrase):
|
|||
|
|
s = phrase.lower().replace('’', "'")
|
|||
|
|
s = re.sub(r'\s+', ' ', s).strip()
|
|||
|
|
return s
|
|||
|
|
|
|||
|
|
|
|||
|
|
PLACE = [("sb.'s", '<ONESP>'), ("one's", '<ONESP>'), ('sb.', '<SB>'), ('sth.', '<STH>'), ('…', '<OBJ>')]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def variant_key(phrase):
|
|||
|
|
"""去掉占位符与句首 be,用于把同一搭配的多种书写形式归到一组"""
|
|||
|
|
s = norm(phrase)
|
|||
|
|
for a, b in PLACE:
|
|||
|
|
s = s.replace(a, ' ')
|
|||
|
|
s = re.sub(r'^(be|to)\s+', '', s)
|
|||
|
|
s = re.sub(r'\s+', ' ', s).strip(' .')
|
|||
|
|
return s
|
|||
|
|
|
|||
|
|
|
|||
|
|
def placeholder_kind(phrase):
|
|||
|
|
kinds = []
|
|||
|
|
if '…' in phrase:
|
|||
|
|
kinds.append('…')
|
|||
|
|
if re.search(r"sb\.|sth\.", phrase):
|
|||
|
|
kinds.append('sb./sth.')
|
|||
|
|
if re.search(r"one's|sb\.'s", phrase):
|
|||
|
|
kinds.append("one's")
|
|||
|
|
return '+'.join(kinds) if kinds else None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def classify(phrase):
|
|||
|
|
s = norm(phrase)
|
|||
|
|
toks = s.split(' ')
|
|||
|
|
wc = len(toks)
|
|||
|
|
head, tail = toks[0], toks[-1].strip('.,')
|
|||
|
|
hp = head_pos_guess(head, wc)
|
|||
|
|
tc = tail_mark(tail)
|
|||
|
|
if head == 'be':
|
|||
|
|
st = '系表结构'
|
|||
|
|
elif head in PREP_LEAD:
|
|||
|
|
st = '介词框架'
|
|||
|
|
elif tc == '小品词':
|
|||
|
|
st = '动/形+小品词'
|
|||
|
|
elif tc == '介词':
|
|||
|
|
st = ('动词' if hp in ('动词', '其他') else hp) + '+介词'
|
|||
|
|
elif wc == 1:
|
|||
|
|
st = '单词/复合词'
|
|||
|
|
else:
|
|||
|
|
st = '其他'
|
|||
|
|
return dict(normalized=s, variant_key=variant_key(phrase), head=head, head_pos_guess=hp,
|
|||
|
|
structure=st, tail=tail, tail_class=tc, word_count=wc,
|
|||
|
|
has_placeholder=1 if placeholder_kind(phrase) else 0,
|
|||
|
|
placeholder_kind=placeholder_kind(phrase))
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 匹配模式
|
|||
|
|
def inflections(v):
|
|||
|
|
out = {v}
|
|||
|
|
if v.endswith('e'):
|
|||
|
|
out |= {v + 's', v + 'd', v[:-1] + 'ing'}
|
|||
|
|
elif v.endswith('y') and len(v) > 1 and v[-2] not in 'aeiou':
|
|||
|
|
out |= {v[:-1] + 'ies', v[:-1] + 'ied', v + 'ing'}
|
|||
|
|
else:
|
|||
|
|
out |= {v + 's', v + 'ed', v + 'ing'}
|
|||
|
|
if len(v) > 3 and v[-1] not in 'aeiouy' and v[-2] in 'aeiou' and v[-3] not in 'aeiou':
|
|||
|
|
out |= {v + v[-1] + 'ing', v + v[-1] + 'ed'}
|
|||
|
|
return sorted(out)
|
|||
|
|
|
|||
|
|
|
|||
|
|
OBJ = r"[a-z'\-]{1,18}(?:\s+[a-z'\-]{1,18}){0,2}"
|
|||
|
|
SUB = r"(?:sb\.|someone|somebody|us|him|her|them|me|you)"
|
|||
|
|
THG = r"(?:sth\.|something|it|them|that)"
|
|||
|
|
POSS = r"[a-z]+'s"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_pattern(phrase, hp):
|
|||
|
|
s = norm(phrase)
|
|||
|
|
for a, b in PLACE:
|
|||
|
|
s = s.replace(a, b)
|
|||
|
|
esc = re.escape(s)
|
|||
|
|
toks = esc.split(' ')
|
|||
|
|
if toks and toks[0] == 'be':
|
|||
|
|
toks[0] = r'(?:be|is|are|was|were|been|being|am)'
|
|||
|
|
elif toks and re.fullmatch(r'[A-Za-z]+', toks[0]) and hp == '动词' and len(toks) > 1:
|
|||
|
|
toks[0] = '(?:' + '|'.join(inflections(toks[0])) + ')'
|
|||
|
|
toks = ['(?:do|does|did|done|doing)' if t == 'do' else t for t in toks]
|
|||
|
|
s = ' '.join(toks)
|
|||
|
|
s = s.replace('<ONESP>', POSS).replace('<SB>', SUB).replace('<STH>', THG).replace('<OBJ>', OBJ)
|
|||
|
|
return re.compile(r'\b' + s + r'\b', re.I)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sent_of(text, a, b):
|
|||
|
|
L = text.rfind('.', max(0, a - 250), a)
|
|||
|
|
R = text.find('.', b, b + 250)
|
|||
|
|
s = text[(L + 1 if L != -1 else max(0, a - 130)):(R + 1 if R != -1 else b + 130)]
|
|||
|
|
return re.sub(r'\s+', ' ', s).strip()[:320]
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 语料
|
|||
|
|
def corpus_docs():
|
|||
|
|
out = []
|
|||
|
|
for sub in ('英语一', '英语二'):
|
|||
|
|
d = os.path.join(PDFROOT, sub)
|
|||
|
|
if not os.path.isdir(d):
|
|||
|
|
continue
|
|||
|
|
for fn in sorted(os.listdir(d)):
|
|||
|
|
if not fn.lower().endswith('.pdf') or '带书签' in fn or '无解析' in fn:
|
|||
|
|
continue
|
|||
|
|
m = re.search(r'(?:19|20)\d{2}', fn)
|
|||
|
|
if not m:
|
|||
|
|
continue
|
|||
|
|
out.append((int(m.group(0)), sub, fn, os.path.join(d, fn)))
|
|||
|
|
out.sort()
|
|||
|
|
return out
|
|||
|
|
|
|||
|
|
|
|||
|
|
def pdf_text(p):
|
|||
|
|
doc = pymupdf.open(p)
|
|||
|
|
t = ''.join(pg.get_text() for pg in doc)
|
|||
|
|
doc.close()
|
|||
|
|
t = t.replace('\u00ad', '').replace('\u2019', "'").replace('\u2018', "'").replace('\u2014', ' - ')
|
|||
|
|
return re.sub(r'\s+', ' ', t)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------- 建库
|
|||
|
|
def main():
|
|||
|
|
raw = [l.strip() for l in io.open(SRC, encoding='utf-8') if l.strip()]
|
|||
|
|
rows = []
|
|||
|
|
seen = set()
|
|||
|
|
for p in raw:
|
|||
|
|
if p in seen:
|
|||
|
|
continue
|
|||
|
|
seen.add(p)
|
|||
|
|
rows.append((p, classify(p)))
|
|||
|
|
print('词表:%d 条(去重后 %d)' % (len(raw), len(rows)))
|
|||
|
|
|
|||
|
|
docs = corpus_docs()
|
|||
|
|
print('语料:%d 份真题 PDF,抽取中…' % len(docs))
|
|||
|
|
texts = []
|
|||
|
|
for i, (y, t, fn, path) in enumerate(docs, 1):
|
|||
|
|
txt = pdf_text(path)
|
|||
|
|
texts.append((y, t, fn, txt))
|
|||
|
|
print(' [%2d/%d] %d %s %d 字符' % (i, len(docs), y, t, len(txt)), flush=True)
|
|||
|
|
|
|||
|
|
if os.path.exists(DB):
|
|||
|
|
os.remove(DB)
|
|||
|
|
con = sqlite3.connect(DB)
|
|||
|
|
con.executescript('''
|
|||
|
|
PRAGMA foreign_keys=ON;
|
|||
|
|
CREATE TABLE sources(
|
|||
|
|
source_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|||
|
|
list_name TEXT NOT NULL, repo TEXT NOT NULL, repo_url TEXT NOT NULL,
|
|||
|
|
file_path TEXT NOT NULL, license TEXT NOT NULL, license_note TEXT,
|
|||
|
|
entry_count INTEGER NOT NULL, retrieved_at TEXT NOT NULL,
|
|||
|
|
UNIQUE(list_name, repo, file_path));
|
|||
|
|
CREATE TABLE phrases(
|
|||
|
|
phrase_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|||
|
|
phrase TEXT NOT NULL,
|
|||
|
|
normalized TEXT NOT NULL,
|
|||
|
|
variant_key TEXT NOT NULL,
|
|||
|
|
head TEXT, head_pos_guess TEXT,
|
|||
|
|
structure TEXT, tail TEXT, tail_class TEXT,
|
|||
|
|
word_count INTEGER NOT NULL,
|
|||
|
|
has_placeholder INTEGER NOT NULL, placeholder_kind TEXT,
|
|||
|
|
source_id INTEGER NOT NULL REFERENCES sources(source_id),
|
|||
|
|
list_name TEXT NOT NULL,
|
|||
|
|
variant_count INTEGER DEFAULT 1,
|
|||
|
|
exam_hits INTEGER DEFAULT 0,
|
|||
|
|
exam_docs TEXT, first_year INTEGER,
|
|||
|
|
tags TEXT,
|
|||
|
|
cn_meaning TEXT, usage_note TEXT, importance INTEGER,
|
|||
|
|
review_status TEXT DEFAULT '未复习',
|
|||
|
|
created_at TEXT NOT NULL,
|
|||
|
|
UNIQUE(phrase, list_name));
|
|||
|
|
CREATE TABLE corpus_docs(
|
|||
|
|
doc_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|||
|
|
exam_year INTEGER NOT NULL, exam_type TEXT NOT NULL,
|
|||
|
|
source_file TEXT NOT NULL, word_count INTEGER, char_count INTEGER, text TEXT,
|
|||
|
|
UNIQUE(exam_year, exam_type));
|
|||
|
|
CREATE TABLE occurrences(
|
|||
|
|
occ_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|||
|
|
phrase_id INTEGER NOT NULL REFERENCES phrases(phrase_id),
|
|||
|
|
doc_id INTEGER NOT NULL REFERENCES corpus_docs(doc_id),
|
|||
|
|
matched_text TEXT, sentence TEXT);
|
|||
|
|
CREATE INDEX idx_ph_head ON phrases(head);
|
|||
|
|
CREATE INDEX idx_ph_variant ON phrases(variant_key);
|
|||
|
|
CREATE INDEX idx_ph_structure ON phrases(structure);
|
|||
|
|
CREATE INDEX idx_ph_exam ON phrases(exam_hits);
|
|||
|
|
CREATE INDEX idx_occ_phrase ON occurrences(phrase_id);
|
|||
|
|
CREATE INDEX idx_occ_doc ON occurrences(doc_id);
|
|||
|
|
''')
|
|||
|
|
|
|||
|
|
cur = con.cursor()
|
|||
|
|
cur.execute('''INSERT INTO sources(list_name,repo,repo_url,file_path,license,license_note,entry_count,retrieved_at)
|
|||
|
|
VALUES(?,?,?,?,?,?,?,?)''',
|
|||
|
|
('考研', REPO, URL, 'lists/npee.txt', LIC,
|
|||
|
|
'fork of moodHappy/english-phrases(上游考研表仅 875 条)', len(rows), NOW))
|
|||
|
|
sid = cur.lastrowid
|
|||
|
|
|
|||
|
|
for y, t, fn, txt in texts:
|
|||
|
|
cur.execute('''INSERT INTO corpus_docs(exam_year,exam_type,source_file,word_count,char_count,text)
|
|||
|
|
VALUES(?,?,?,?,?,?)''', (y, t, fn, len(txt.split()), len(txt), txt))
|
|||
|
|
doc_ids = [r[0] for r in cur.execute('SELECT doc_id FROM corpus_docs ORDER BY doc_id')]
|
|||
|
|
|
|||
|
|
print('逐条检索中(%d 条 × %d 卷)…' % (len(rows), len(texts)))
|
|||
|
|
hit_phrases = 0
|
|||
|
|
total_occ = 0
|
|||
|
|
for phrase, info in rows:
|
|||
|
|
cs = classify(phrase) # 与 rows 中一致,取词性供模式构建
|
|||
|
|
pat = build_pattern(phrase, cs['head_pos_guess'])
|
|||
|
|
info = dict(info)
|
|||
|
|
cur.execute('''INSERT INTO phrases(phrase,normalized,variant_key,head,head_pos_guess,structure,tail,
|
|||
|
|
tail_class,word_count,has_placeholder,placeholder_kind,source_id,list_name,created_at)
|
|||
|
|
VALUES(:phrase,:normalized,:variant_key,:head,:head_pos_guess,:structure,:tail,
|
|||
|
|
:tail_class,:word_count,:has_placeholder,:placeholder_kind,:sid,:ln,:now)''',
|
|||
|
|
dict(info, phrase=phrase, sid=sid, ln='考研', now=NOW))
|
|||
|
|
pid = cur.lastrowid
|
|||
|
|
hits, dlabels, years = 0, set(), []
|
|||
|
|
for (y, t, fn, txt), did in zip(texts, doc_ids):
|
|||
|
|
n = 0
|
|||
|
|
for m in pat.finditer(txt):
|
|||
|
|
n += 1
|
|||
|
|
total_occ += 1
|
|||
|
|
if hits < 3:
|
|||
|
|
cur.execute('INSERT INTO occurrences(phrase_id,doc_id,matched_text,sentence) VALUES(?,?,?,?)',
|
|||
|
|
(pid, did, m.group(0)[:60], sent_of(txt, m.start(), m.end())))
|
|||
|
|
hits += 1
|
|||
|
|
if n >= 8:
|
|||
|
|
break
|
|||
|
|
if n:
|
|||
|
|
dlabels.add('%d%s' % (y, '英一' if t == '英语一' else '英二'))
|
|||
|
|
years.append(y)
|
|||
|
|
if hits:
|
|||
|
|
hit_phrases += 1
|
|||
|
|
cur.execute('UPDATE phrases SET exam_hits=?, exam_docs=?, first_year=? WHERE phrase_id=?',
|
|||
|
|
(hits, ','.join(sorted(dlabels)), min(years), pid))
|
|||
|
|
|
|||
|
|
# 变体计数
|
|||
|
|
cur.execute('''UPDATE phrases SET variant_count=(SELECT COUNT(*) FROM phrases p2
|
|||
|
|
WHERE p2.variant_key=phrases.variant_key)''')
|
|||
|
|
|
|||
|
|
# 标签种子:2010 真题完形做错的 4 个搭配
|
|||
|
|
WRONG2010 = {'be subjected to': '2010完形错题', 'subject to': '2010完形错题',
|
|||
|
|
'end up': '2010完形错题', 'end up doing': '2010完形错题',
|
|||
|
|
'so long as': '2010完形错题', 'as long as': '2010完形错题',
|
|||
|
|
'contrary to': '2010完形错题', 'on the contrary': '2010完形错题'}
|
|||
|
|
for k, tag in WRONG2010.items():
|
|||
|
|
cur.execute('UPDATE phrases SET tags=? WHERE variant_key=?', (tag, k))
|
|||
|
|
|
|||
|
|
# 视图
|
|||
|
|
con.executescript('''
|
|||
|
|
CREATE VIEW v_particle_stats AS
|
|||
|
|
SELECT tail_class, tail, COUNT(*) AS n FROM phrases GROUP BY tail_class, tail ORDER BY n DESC;
|
|||
|
|
CREATE VIEW v_structure_stats AS
|
|||
|
|
SELECT structure, COUNT(*) AS n FROM phrases GROUP BY structure ORDER BY n DESC;
|
|||
|
|
CREATE VIEW v_variants AS
|
|||
|
|
SELECT variant_key, COUNT(*) AS n, GROUP_CONCAT(phrase, ' | ') AS forms
|
|||
|
|
FROM phrases GROUP BY variant_key HAVING n>1 ORDER BY n DESC;
|
|||
|
|
CREATE VIEW v_exam_phrases AS
|
|||
|
|
SELECT phrase, structure, exam_hits, first_year, exam_docs FROM phrases
|
|||
|
|
WHERE exam_hits>0 ORDER BY exam_hits DESC, phrase;
|
|||
|
|
CREATE VIEW v_todo_cn AS
|
|||
|
|
SELECT phrase_id, phrase, structure FROM phrases WHERE cn_meaning IS NULL ORDER BY phrase;
|
|||
|
|
''')
|
|||
|
|
con.commit()
|
|||
|
|
|
|||
|
|
# 汇总
|
|||
|
|
n_ph = cur.execute('SELECT COUNT(*) FROM phrases').fetchone()[0]
|
|||
|
|
n_doc = cur.execute('SELECT COUNT(*) FROM corpus_docs').fetchone()[0]
|
|||
|
|
n_occ = cur.execute('SELECT COUNT(*) FROM occurrences').fetchone()[0]
|
|||
|
|
print()
|
|||
|
|
print('=' * 66)
|
|||
|
|
print('库文件 : %s (%.1f KB)' % (DB, os.path.getsize(DB) / 1024))
|
|||
|
|
print('phrases : %d 条' % n_ph)
|
|||
|
|
print('corpus_docs : %d 份真题' % n_doc)
|
|||
|
|
print('occurrences : %d 条命中样本' % n_occ)
|
|||
|
|
print('真题命中搭配: %d 条(占 %.1f%%)' % (hit_phrases, 100.0 * hit_phrases / n_ph))
|
|||
|
|
print('命中总次数 : %d' % total_occ)
|
|||
|
|
print('-' * 66)
|
|||
|
|
print('结构分布:')
|
|||
|
|
for s, n in cur.execute('SELECT structure, COUNT(*) FROM phrases GROUP BY structure ORDER BY 2 DESC'):
|
|||
|
|
print(' %-16s %5d (%.1f%%)' % (s, n, 100.0 * n / n_ph))
|
|||
|
|
print('-' * 66)
|
|||
|
|
print('真题命中 Top 15:')
|
|||
|
|
for r in cur.execute('SELECT phrase, exam_hits, exam_docs FROM v_exam_phrases LIMIT 15'):
|
|||
|
|
print(' %-32s %3d 次 %s' % (r[0], r[1], (r[2] or '')[:48]))
|
|||
|
|
print('-' * 66)
|
|||
|
|
print('变体组最多的 10 组:')
|
|||
|
|
for r in cur.execute('SELECT variant_key, n FROM v_variants LIMIT 10'):
|
|||
|
|
print(' %-30s %d 种形式' % (r[0], r[1]))
|
|||
|
|
con.close()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
main()
|