001-process-document-for-candidates.py

# Import the re module for regular expressions
import re
import os
import json


from icecream import ic

SOURCE_DOCUMENTS_DIRECTORY = "source-documents"
INPUT_FILENAME = "Talks-with-Ramana-full-text.txt"
OUTPUT_FILENAME = "001-process-document-for-candidates-output.json"

input_document = os.path.join(SOURCE_DOCUMENTS_DIRECTORY, INPUT_FILENAME)

# Define the regex pattern to match lines with 20 or more non-whitespace characters
pattern = re.compile(r"\S{17}")

# Define an empty list to store the matching lines as dictionaries
matches = []

with open(input_document, "r", encoding="utf8") as file:
    for line, text in enumerate(file, start=1):
        pattern_match = pattern.search(text)
        if pattern_match:
            match_string = pattern_match.group()
            match = {
                "line_number": line,
                "match": match_string,
                "content": text.strip()
                }
            # ic(match)
            matches.append(match)

json_string = json.dumps(matches, indent=4)

# Open the file "output.json" for writing
with open(OUTPUT_FILENAME, "w", encoding="utf8") as file:
    # Write the JSON string to the file
    file.write(json_string)
ic(json_string)