Cleaning Catherine de Medici’s letters

Getting the Letters

I started this process by getting the raw OCR from the Letters. I used the pymupdf library, imported as fitz, to extract the text from the PDF. Pymupdf is great at reading formatted text from a PDF, especially of the kind in the Letters of Catherine de Medici, which is formatted into columns. Luckily, the

import fitz
def extract(path):
    doc = fitz.open(path)
    letters = []
    for page_num in range(len(doc)):
        page = doc.load_page(page_num)
        text = page.get_text("text")
        text = text.replace("\n", " ")
        if "INTRODUCTION." in text:
            continue
        if text.split("\n")[0].startswith("TABLE"):
            break
        split_text = re.split(r"(?<!\.)(?=\d{4}\.)", text)
        if len(letters) == 0:
            letters.extend([f"<{str(page_num +1)}>" + s for s in split_text])
        else:
            letters[-1] += f"<{page_num}> {split_text[0]}"
            letters.extend([f"<{str(page_num +1)}>" + s for s in split_text[1:]])
    else:
        print("didn't break at the tables")
    return letters

Then I had to extract

lines = open("./cdm4.bk.txt").readlines()

for i, line in enumerate(lines):
    if len(line) < 100 and ("CATHERINE" in line or "CATERINE" in line):
        line = re.sub(r"<\d{3}>", " ", line)
        lines[i-1] + line
        lines.pop(i)

with open("./cdm4.bk.txt", "w") as f:
    f.writelines(lines)