74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
import json
|
|||
|
|
import os
|
||
|
|
|
||
|
|
def extract_text_from_content(content_list):
|
||
|
|
"""Recursively extract text from content structure"""
|
||
|
|
result = []
|
||
|
|
for item in content_list:
|
||
|
|
if isinstance(item, dict):
|
||
|
|
if item.get('type') == 'text' and 'content' in item:
|
||
|
|
text = item['content']
|
||
|
|
if isinstance(text, str):
|
||
|
|
result.append(text)
|
||
|
|
elif 'content' in item:
|
||
|
|
result.extend(extract_text_from_content(item['content']))
|
||
|
|
return result
|
||
|
|
|
||
|
|
def json_to_blocks(json_path):
|
||
|
|
"""Parse JSON and return list of (page_idx, blocks) tuples"""
|
||
|
|
with open(json_path, 'r', encoding='utf-8') as f:
|
||
|
|
data = json.load(f)
|
||
|
|
pages = []
|
||
|
|
for page in data['pages']:
|
||
|
|
pages.append((page['page_idx'], page['blocks']))
|
||
|
|
return pages
|
||
|
|
|
||
|
|
def blocks_to_markdown(pages):
|
||
|
|
"""Convert pages of blocks to Markdown"""
|
||
|
|
md_lines = []
|
||
|
|
for page_idx, blocks in sorted(pages, key=lambda x: x[0]):
|
||
|
|
md_lines.append(f"\n<!-- Page {page_idx + 1} -->\n")
|
||
|
|
for block in blocks:
|
||
|
|
btype = block.get('type', 'text')
|
||
|
|
texts = extract_text_from_content(block.get('content', []))
|
||
|
|
full_text = ''.join(texts).strip()
|
||
|
|
|
||
|
|
if not full_text:
|
||
|
|
continue
|
||
|
|
|
||
|
|
if btype == 'doc_title':
|
||
|
|
level = block.get('level', 1)
|
||
|
|
md_lines.append(f"{'#' * level} {full_text}\n")
|
||
|
|
elif btype == 'text':
|
||
|
|
md_lines.append(full_text + "\n")
|
||
|
|
elif btype in ('header', 'footer', 'page_number'):
|
||
|
|
md_lines.append(f"<!-- {btype}: {full_text} -->\n")
|
||
|
|
else:
|
||
|
|
md_lines.append(full_text + "\n")
|
||
|
|
return '\n'.join(md_lines)
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
base_dir = r'D:\mineru_home\doclib\parsed\d9\d991db7dc971c6655661492bb28129f2089d03222b439064e30b7156ff1ba848\flash'
|
||
|
|
|
||
|
|
json_files = [
|
||
|
|
os.path.join(base_dir, '1-10_1794301361022.json'),
|
||
|
|
os.path.join(base_dir, '11-318_1794301886872.json'),
|
||
|
|
]
|
||
|
|
|
||
|
|
all_pages = []
|
||
|
|
for jf in json_files:
|
||
|
|
pages = json_to_blocks(jf)
|
||
|
|
all_pages.extend(pages)
|
||
|
|
print(f"Loaded {jf}: {len(pages)} pages")
|
||
|
|
|
||
|
|
print(f"Total pages: {len(all_pages)}")
|
||
|
|
|
||
|
|
md = blocks_to_markdown(all_pages)
|
||
|
|
|
||
|
|
output_path = r'D:\考研\期末考\无损检测\ocr_work\full_book.md'
|
||
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
||
|
|
f.write(md)
|
||
|
|
|
||
|
|
print(f"Markdown saved to: {output_path}")
|
||
|
|
print(f"Total chars: {len(md)}")
|