monitor.py

import json
import shelve
from flask import Flask, render_template, request, jsonify
from openai import OpenAI
from diffstrings import diff_strings, SequenceMatcher
from markupsafe import Markup
from icecream import ic
import difflib
from difflib import Differ
from html import escape
    
app = Flask(__name__)

# Configuration
INPUT_FILE = "001-process-document-for-candidates-output.json"
SHELF_FILE = "processing_progress.shelf"

client = OpenAI(base_url="http://localhost:1234/v1", api_key="not-needed")

def get_ai_response(prompt, line_content):
    try:
        completion = client.chat.completions.create(
            model="local-model",
            messages=prompt + [{"role": "user", "content": line_content}],
            temperature=0.1,
            max_tokens=(len(line_content) + 20),
            top_p=0.95,
        )
        return completion.choices[0].message.content
    except Exception as e:
        print(f"Error getting AI response: {e}")
        return None




# @app.route('/')
# def index():
#     with open(INPUT_FILE, 'r') as f:
#         data = json.load(f)
#     with shelve.open(SHELF_FILE) as shelf:
#         for item in data:
#             line_number = str(item['line_number'])
#             if line_number in shelf:
#                 item['processed'] = True
#                 item['corrected_content'] = shelf[line_number]['corrected_content']
#                 # diffstrings method
#                 # s = s = SequenceMatcher(None, item['corrected_content'], item['content'], autojunk=False)
#                 # item['diff'] = s.diff_strings(True)
#                 # differ method
#                 item['diff'] = diff_texts(item['content'], item['corrected_content'])
#                 ic(item['content'])
#                 ic(item['diff'])
#             else:
#                 item['processed'] = False
#                 item['corrected_content'] = ''
#                 item['diff'] = ''
#     return render_template('index.html', items=data)
@app.route('/')
def index():
    try:
        with open(INPUT_FILE, 'r') as f:
            data = json.load(f)
        with shelve.open(SHELF_FILE) as shelf:
            for item in data:
                line_number = str(item['line_number'])
                if line_number in shelf:
                    item['processed'] = True
                    item['corrected_content'] = shelf[line_number]['corrected_content']
                    item['diff'] = diff_texts(item['content'], item['corrected_content'])
                    item['modified_original'] = apply_diff_to_original(item['content'], item['diff'])
                    item['modified_corrected'] = apply_diff_to_corrected(item['corrected_content'], item['diff'])
                    ic(item['content'])
                    ic(item['diff'])
                else:
                    item['processed'] = False
                    item['corrected_content'] = ''
                    item['diff'] = []
                    item['modified_original'] = escape(item['content'])
                    item['modified_corrected'] = ''
        return render_template('index.html', items=data)
    except Exception as e:
        return f"An error occurred: {str(e)}", 500

def format_diff(diff):
    formatted = []
    for text, op in diff:
        if op == 0:  # No change
            formatted.append(text)
        elif op == 1:  # Addition
            formatted.append(f'<span class="diff-add">{text}</span>')
        elif op == -1:  # Deletion
            formatted.append(f'<span class="diff-del">{text}</span>')
    return Markup(''.join(formatted))

# def diff_texts(text1="", text2=""):
#     # Initiate the differ object
#     d = Differ()
#     # Calculate the difference
#     return [
#         (f"{token[2:]} " if token[0] != " " else "",
#          token[0] if token[0] != " " else None)
#         # Show word-level granularity for difference
#         for token in d.compare(text1.split(), text2.split())

#         # (token[2:], token[0] if token[0] != " " else None)
#         # Show world-level granularity for difference. Remove .split() to
#         # return to character-level granularity
#         # for token in d.compare(text1.split(), text2.split())
#     ]
def diff_texts(text1="", text2=""):
    def group_opcodes(opcodes, n=4):
        group = []
        for opcode in opcodes:
            group.append(opcode)
            if len(group) == n:
                yield group
                group = []
        if group:
            yield group

    matcher = difflib.SequenceMatcher(None, text1, text2)
    grouped_opcodes = list(group_opcodes(matcher.get_opcodes()))
    
    result = []
    for group in grouped_opcodes:
        for tag, i1, i2, j1, j2 in group:
            if tag == 'equal':
                result.append(('equal', text1[i1:i2]))
            elif tag == 'delete':
                result.append(('delete', text1[i1:i2]))
            elif tag == 'insert':
                result.append(('insert', text2[j1:j2]))
            elif tag == 'replace':
                result.append(('delete', text1[i1:i2]))
                result.append(('insert', text2[j1:j2]))
    return result
def apply_diff_to_original(text, diff):
    result = []
    for op, content in diff:
        if op == 'equal':
            result.append(escape(content))
        elif op == 'delete':
            result.append(f'<span class="diff-del">{escape(content)}</span>')
    return Markup(''.join(result))

def apply_diff_to_corrected(text, diff):
    result = []
    for op, content in diff:
        if op == 'equal':
            result.append(escape(content))
        elif op == 'insert':
            result.append(f'<span class="diff-add">{escape(content)}</span>')
    return Markup(''.join(result))

@app.route('/item/<int:line_number>')
def item_detail(line_number):
    with open(INPUT_FILE, 'r') as f:
        data = json.load(f)
    original_item = next((item for item in data if item['line_number'] == line_number), None)
    with shelve.open(SHELF_FILE) as shelf:
        processed_item = shelf.get(str(line_number))
    return render_template('item_detail.html', original=original_item, processed=processed_item)

@app.route('/update', methods=['POST'])
def update_item():
    line_number = request.form['line_number']
    corrected_content = request.form['corrected_content']
    with shelve.open(SHELF_FILE, writeback=True) as shelf:
        item = shelf[line_number]
        item['corrected_content'] = corrected_content
        shelf[line_number] = item
    return jsonify(success=True)

@app.route('/delete', methods=['POST'])
def delete_item():
    line_number = request.form['line_number']
    with shelve.open(SHELF_FILE, writeback=True) as shelf:
        if line_number in shelf:
            del shelf[line_number]
    return jsonify(success=True)

@app.route('/process', methods=['POST'])
def process_item():
    line_number = request.form['line_number']
    with open(INPUT_FILE, 'r') as f:
        data = json.load(f)
    item = next((item for item in data if str(item['line_number']) == line_number), None)
    if item:
        prompt = [
            {
                "role": "system",
                "content": "You are a professional proofreader specializing in correcting text where words may be incorrectly joined together. Your task is to insert spaces and/or punctuation where needed to correct the text. Please provide only the corrected text in your response.",
            }
        ]
        corrected_content = get_ai_response(prompt, item['content'])
        if corrected_content:
            with shelve.open(SHELF_FILE, writeback=True) as shelf:
                item['corrected_content'] = corrected_content
                shelf[line_number] = item
            return jsonify(success=True, corrected_content=corrected_content)
    return jsonify(success=False)


if __name__ == '__main__':
    app.run(debug=True)