proofread.py

#!/usr/bin/env python3
# -*- coding: utf-8  -*-
#
# Reza (User:reza1615), 2011
# Huji (User:Huji), 2025
#
# Distributed under the terms of the MIT license.
#
import sys
import os
import json
import re
import time
import requests
import urllib.parse
from operator import itemgetter
import suggest
import logging


log = logging.getLogger('logger')


def read_data(file_name: str):
    """Reads reference data from a file in the /data directory

    Args:
        file_name (str): Name of the file

    Returns:
        str: Content of the file
    """
    base_path = os.getcwd() + "/"
    f = open(base_path + "data/" + file_name, 'r')
    data = f.read().strip()
    f.close()
    return data


def get_page_wikitext(title: str) -> str:
    """Fetches the source wikitext of a page on Persian Wikipedia.

    Args:
        title (str): Title of the page.

    Returns:
        str: Sourece wikitext of the page.
    """
    txt = ""
    try:
        txt = list(
            requests.get(
                "https://fa.wikipedia.org/w/api.php",
                params={
                    "titles": title,
                    "action": "query",
                    "prop": "revisions",
                    "rvprop": "content",
                    "format": "json",
                    "rvslots": "main",
                },
            )
            .json()["query"]["pages"]
            .values()
        )[0]["revisions"][0]["slots"]["main"]["*"]
    except Exception:
        time.sleep(2)
        try:
            txt = list(
                requests.get(
                    "https://fa.wikipedia.org/w/api.php",
                    params={
                        "titles": title,
                        "action": "query",
                        "prop": "revisions",
                        "rvprop": "content",
                        "format": "json",
                        "rvslots": "main",
                    },
                )
                .json()["query"]["pages"]
                .values()
            )[0]["revisions"][0]["slots"]["main"]["*"]
        except Exception:
            # TODO: better error handling
            return None

    return txt


def get_page_html(title: str) -> str:
    """Fetches the HTML output of a page on Persian Wikipedia.

    Args:
        title (str):Title of the page

    Returns:
        str: HTML output of the page
    """
    url = "https://fa.wikipedia.org/wiki/" + urllib.parse.quote(title)
    req = requests.get(url)
    return req.text


def update_ref_data(restart_webservice=False) -> str:
    """Updates specific reference data files based on the content of
    corresponding pages on Wikipedia.

    Args:
        restart_webservice (bool, optional): Whether to restart the webserver.
        Defaults to False.

    Returns:
        str: Final status of the update process, in JSON format.
    """
    base_path = os.getcwd() + "/"

    map = {
        "user_provided_blacklist.txt": "ویکی‌پدیا:اشتباه‌یاب/فهرست",
        "user_provided_whitelist.txt": "ویکی‌پدیا:اشتباه‌یاب/فهرست موارد درست"
    }

    try:
        for file, page in map.items():
            f = open(base_path + "data/%s" % file, "r")
            current = f.read()
            f.close()
            updated = get_page_wikitext(page).replace("{{/بالا}}", "") \
                .lstrip("\n")

            if current != updated:
                f = open(base_path + "data/%s" % file, "w")
                f.write(updated)
                f.close()

        output = "{\"response\": \"Update successful\"}"
    except Exception:
        output = "{\"response\": \"Update failed\"}"

    if restart_webservice:
        os.system('toolforge webservice restart')

    return output


def process_ambiguous_links(html: str) -> list[dict]:
    """Find ambibuous links in the provided HTML. This function ignores
    ambiguous links that are created by disambig templates. For each
    ambiguous link it finds, it attemps to suggest alternative links by
    retrieving the titles of pages that were linked from the ambiguous
    link target.

    Args:
        html (str): HTML output of a Wikipedia page

    Returns:
        list[dict]: A list of dictionaries, with each item representing a
                    proofread output item.
    """
    ambig_titles = []
    output = []

    # ignore ambiguous links that are generated by disambig templates
    html = re.sub(
        r'<div [^>]*?role="note"[^>]*?>[^<]*<a[^>]*?class="mw-disambig"[^>]*?>[^>]*?<\/a>[^>]*?<\/div>', "", html
    )

    pat = re.compile(r"<a [^>]*?(class=\"mw-disambig\" [^>]*?title=\"[^\"]+?\"|" +  # noqa:W504
                     r"title=\"[^\"]+?\" [^>]*?class=\"mw-disambig\")" +  # noqa:W504
                     r"[^>]*?>.*?<\/a>")

    links = pat.findall(html)

    for link in links:
        title = re.findall(r"title=\"(.+?)\"", link)[0]

        if title not in ambig_titles:
            suggestions = get_disambig_candidates(title)
            ambig_titles.append(title)
            output.append({
                "type": 8,
                "word": title,
                "cleaned_word": title,
                "suggestions": suggestions,
            })

    return output


def get_disambig_candidates(title: str) -> list[str]:
    """Given the title of a dismabiguation page, suggests candidate pages that
    can be used instead of it to resolve an ambiguous link. This function only
    returns those pages linked from the diasmbiguation page whose title
    contains the title of the disambiguation page.

    Args:
        title (str): Title of the dismabiguation page

    Returns:
        list[str]: A list of candidate page titles
    """
    output = []
    title = title.replace("_", " ")

    params = {
        "action": "query",
        "prop": "links",
        "titles": title,
        "pllimit": 500,
        "plnamespace": 0,
        "format": "json",
    }

    try:
        links = list(
            requests.post("https://fa.wikipedia.org/w/api.php", params)
            .json()["query"]["pages"]
            .values()
        )[0]["links"]
    except Exception:
        time.sleep(2)
        try:
            links = list(
                requests.post("https://fa.wikipedia.org/w/api.php", params)
                .json()["query"]["pages"]
                .values()
            )[0]["links"]
        except Exception:
            # TODO: better error handling
            links = []

    for link in links:
        # if the original link title doesn't appear in the candidate, ignore it
        if title not in link["title"]:
            continue

        candidate = link["title"]

        m = re.match(r"^([^()]+) \(.+\)$", candidate)

        # if the candidate's title ends with a parenthetical, wikify it
        if m is not None:
            candidate = candidate + "|" + m[1]

        output.append(candidate)

    return output


def suggest_spaced_conjunction(word: str) -> str:
    """For words that appear to start with a prefx, suggests alternate
    dictations in which the prefix is separate by a space from the
    rest of the word.

    Args:
        word (str): The original word

    Returns:
        str: A phrase that starts with the prefix followed by a space followed
        by the rest of the original word
    """
    pat = re.compile(r'^(و|در|با|هم|از|که|هزار)(\S.*)$')
    if suggest.valid(word):
        return None
    m = pat.match(word)
    if m and suggest.valid(m[2]):
        return m[1] + " " + m[2]

def clean_wikitext(wikitext: str) -> str:
    """Remove some wiki syntax from wikitext. This helps with finding the corresponding
    piece of text on the screen by the JS code.

    Args:
        wikitext (str): Some wikitext

    Returns:
        str: The output string
    """
    pat = "\[\[([^|]+)\|([^\]]+)\]\]"
    wikitext = re.sub(pat, "\\2", wikitext)

    pat = "\[\[([^\]]+)\]\]"
    wikitext = re.sub(pat, "\\1", wikitext)

    return wikitext


def find_asymmetrical_quotation_marks(wikitxt: str) -> list[str]:
    """Find cases of asymmetrical Persian quotation marks. For each case,
    the function generates a brief output consisting only of as little text as
    necessary for the user to identify those segments in the text of the page.

    Args:
        wikitxt (str): Some wikitext

    Returns:
        list[str]: A list of strings representing each case of asymmeterical
        quotation marks.

    """
    output = []

    # find all cases of asymterical quotation marks
    for line in wikitxt.split("\n"):
        if line.count("«") > line.count("»"):
            found = re.findall(r"(?!«[^»«]+»)(«[^»«]+)", line)
            for f in found:
                f = clean_wikitext(f)
                output.append(f)
        elif line.count("«") < line.count("»"):
            line = line[::-1]
            found = re.findall(r"(?!»[^»«]+«)(»[^»«]+)", line)
            for f in found:
                f = f[::-1]
                f = clean_wikitext(f)
                output.append(f)

    # reduce the displayed output to the minimum necessary
    for idx, item in enumerate(output):
        if item[0] == "«":
            if len(item) > 30:
                output[idx] = item[:29]
        else:
            if len(item) > 30:
                output[idx] = item[-29:]

    return output


honorifics_blacklist, most_words_list, Persian_words_list, wiki_titles_list, slang_list, profanity_pat = [], [], [], [], [], []
user_provided_blacklist = {}


def convert_regex(input, new_matches, dict):
    if '?' in input:
        Char_Index = input.find('?')
        new_match = input.replace('?', '')
        if new_match not in new_matches:
            try:
                new_matches.append(new_match)
            except Exception:
                new_matches[new_match] = dict[input]

        new_match = input[0:Char_Index - 1] + input[Char_Index + 1:]
        if new_match not in new_matches:
            try:
                new_matches.append(new_match)
            except Exception:
                new_matches[new_match] = dict[input]

    if re.sub(r'[یک]', '', input) != input:
        new_match = input.replace('ی', 'ي')
        if new_match not in new_matches:
            try:
                new_matches.append(new_match)
            except Exception:
                new_matches[new_match] = dict[input]
        new_match = input.replace('ک', 'ك')
        if new_match not in new_matches:
            try:
                new_matches.append(new_match)
            except Exception:
                new_matches[new_match] = dict[input]
        new_match = input.replace('ک', 'ك').replace('ی', 'ي')
        if new_match not in new_matches:
            try:
                new_matches.append(new_match)
            except Exception:
                new_matches[new_match] = dict[input]

        new_match = input.replace('ک', '[کك]').replace('ی', '[یي]')
        if new_match not in new_matches:
            try:
                new_matches.append(new_match)
            except Exception:
                new_matches[new_match] = dict[input]
    return new_matches


def load_ref_data():
    # TODO eliminate the need for global variables
    global honorifics_blacklist, user_provided_blacklist, most_words_list, Persian_words_list, wiki_titles_list, slang_list, profanity_pat
    honorifics_blacklist, most_words_list, Persian_words_list, wiki_titles_list, slang_list, profanity_pat = [], [], [], [], [], []

    add_regex2 = {}
    add_regex = []
    user_provided_blacklist = {}

    base_path = os.getcwd() + "/"

    with open(base_path + 'data/profanity_regex.txt', 'r') as f:
        profanity_pat = [f.read().strip()]

    with open(base_path + 'data/user_provided_blacklist.txt', 'r') as f:
        blacklist = f.read()

    # Extract the rules from user provided blacklist and turn them into regex
    blacklist = blacklist.replace('\\(', '(').replace('\\)', ')')

    lines = blacklist.split('\n')

    for line in lines:
        # ignore seciton headings and comment lines
        if line.strip()[0:1] in ['#', '=']:
            continue

        # Process the section for disallowed honorific prefixes and suffixes
        if line.strip()[0:1] == '*':
            honorific = line.split('||')[0].replace('*', '').strip()
            honorifics_blacklist.append(honorific)
            add_regex = convert_regex(honorific, add_regex, False)

        # Process the section for common typos and suggested corrections
        elif line.startswith(' ') and '||' in line:
            typo = line.split('||')[0].strip()
            suggestion = line.split('||')[1].strip()
            user_provided_blacklist[typo] = suggestion
            add_regex2 = convert_regex(typo, add_regex2, user_provided_blacklist)

    for i in add_regex:
        if i not in honorifics_blacklist:
            honorifics_blacklist.append(i)

    for i in add_regex2:
        if i not in user_provided_blacklist:
            user_provided_blacklist[i] = add_regex2[i]

    with open(base_path + 'data/slang_regex.txt', 'r') as f:
        slang = f.read()

    # --slang
    lines = slang.split('\n')

    for line in lines:
        # ignore seciton headings and comment lines
        if line.strip()[0:1] in ['#', '=']:
            continue

        if line.strip().startswith('*'):  # عامیانه
            line = re.sub(r'^\*', '', line)
            slang_list.append(line)

    with open(base_path + 'data/most_word_dict.txt', 'r') as f:
        most_words = f.read()

    with open(base_path + 'data/user_provided_whitelist.txt', 'r') as f:
        most_words2 = f.read()
        most_words2 = most_words2.replace('* ', '').replace('*', '').replace('\r', '').strip()

    most_words = most_words + '\n' + most_words2
    most_words = most_words.replace('\r', '')
    most_words_list = most_words.split('\n')

    with open(base_path + 'data/Persian_word_dict.txt', 'r') as f:
        Persian_words = f.read()
        Persian_words = Persian_words.replace('\r', '')
        Persian_words_list = Persian_words.split('\n')

    with open(base_path + 'data/wiki_titles_dict.txt', 'r') as f:
        wiki_titles = f.read()
        wiki_titles = wiki_titles.replace('\r', '')
        wiki_titles_list = wiki_titles.split('\n')


def clean_text(wikitext, remove_regex, faTitle):
    Syntaxs_list = []
    Syntaxs_list2 = []
    Erab_words = []
    giumeh_place = []
    fa_syntax_text = re.sub(r'<(nowiki|math|code|pre|source|syntaxhighlight|blockquote)(?:[^<]|<(?!\/\1>))*?<\/\1>', '', wikitext)
    fa_syntax_text = re.sub(r'%7B%7B', '{{', fa_syntax_text)
    fa_syntax_text = re.sub(r'%7D%7D', '}}', fa_syntax_text)
    fa_syntax_text = re.sub(r'\<\!\-\-(?:[^-]|-(?!->))*?\-\-\>', '', fa_syntax_text)
    fa_syntax_text_old = fa_syntax_text
    fa_syntax_text = re.sub(r'\{\{\{[^\}\{]+\}\}\}', '', fa_syntax_text)
    fa_syntax_text = re.sub(r'\{\{\{[^\}\{]+\}\}\}', '', fa_syntax_text)
    fa_syntax_text = re.sub(r'\{\{\{[^\}\{]+\}\}\}', '', fa_syntax_text)
    if fa_syntax_text_old != fa_syntax_text:
        Syntaxs_list.append('استفاده از متغییر الگو مانند {{{ }}} در مقاله')
    if fa_syntax_text.count("{{") > fa_syntax_text.count("}}"):
        Syntaxs_list.append('{{')
    if fa_syntax_text.count("{{") < fa_syntax_text.count("}}"):
        Syntaxs_list.append('}}')
    if fa_syntax_text.count("[[") > fa_syntax_text.count("]]"):
        Syntaxs_list.append('[[')
    if fa_syntax_text.count("[[") < fa_syntax_text.count("]]"):
        Syntaxs_list.append(']]')

    giumeh_place.append(find_asymmetrical_quotation_marks(fa_syntax_text_old))
    if len(giumeh_place) > 0:
        Syntaxs_list2.append('«»')

    if fa_syntax_text.count("<!--") != fa_syntax_text.count("-->"):
        Syntaxs_list.append('<!-- یکی از علامت‌های شروع یا پایان توضیح وجود ندارد -->')

    wikitext = re.sub(r'<(nowiki|math|code|pre|source|syntaxhighlight|blockquote)(?:[^<]|<(?!\/\1>))*?<\/\1>', '', wikitext)
    wikitext = re.sub(r'\[\[[^\[]*?\]\]', ' ', wikitext)
    wikitext = re.sub(r'\{\{(?:عربی|شروع عربی|آغاز عربی)\}\}([\s\S]*?)\{\{(?:پایان عربی)\}\}', '', wikitext)
    wikitext = re.sub(r'\{\{(?:به .+?|به انگلیسی|انگلیسی|عربی|حدیث|به عربی|به اردو|اردو|lang\-[au]r)[\s\S]*?\}\}', '', wikitext)
    wikitext = re.sub(r'\[\[[^\]]\]\]', '', wikitext)
    wikitext = re.sub(r'[�  ᠎             　]', ' ', wikitext)
    wikitext = re.sub(r'\/\/.*?(?=[\s\n\|\}\]<]|$)', ' ', wikitext)  # حذف نشانی اینترنتی
    wikitext = re.sub(r'(\|.*?\=)', ' ', wikitext)
    wikitext = re.sub(r'\[\[رده\:.*?\]\]', ' ', wikitext)
    wikitext = re.sub(r'(\{\{.*?\}\})', ' ', wikitext)
    wikitext = re.sub(r'(\{\{.*?\|)', ' ', wikitext)
    wikitext = re.sub(r'(\<.*?\>)', ' ', wikitext)
    wikitext = re.sub(r'\r', '', wikitext)
    wikitext = re.sub(r"([\^\%\$\#\@\&\,\=\{\[\}\]\'\|۱۲۳۴۵۶۷۸۹۰\?\.\!؟،\:؛\"\/\\\t\'\*\+\–\-\n0-9٬٫a-zA-Z\_\ـ])+", ' ', wikitext)
    isolated_char = 'ﭖﭗﭘﭙﭺﭻﭼﭽﮊﮋﮎﮏﮐﮑﻙﻚﻛﻜﮒﮓﮔﮕﮤﮥﯼﯽﯾﯿﻯﻰﻱﻲﻳﻴﺁﺁﺂﺄﺃﺃﺅﺅﺆﺇﺈﺇﺉﺊﺋﺌﺍﺎﺏﺐﺑﺒﺕﺖﺗﺘﺙﺚﺛﺜﺝﺞﺟﺠﺡﺢﺣﺤﺥﺦﺧﺨﺩﺪﺫﺬﺭﺮﺯﺰﺱﺲﺳﺴﺵﺶﺷﺸﺹﺺﺻﺼﺽﺾﺿﻀﻁﻂﻃﻄﻅﻆﻇﻈﻉﻊﻋﻌﻍﻎﻏﻐﻑﻒﻓﻔﻕﻖﻗﻘﻝﻞﻟﻠﻡﻢﻣﻤﻥﻦﻧﻨﻩﻪﻫﻬﻫﻭﻮﻰﻲﻵﻶﻸﻷﻹﻺﻻﻼ'
    wikitext = re.sub(r'([^ ‌ء-يٓ-ٕپچژگکكڪﻙﻚیﻱﻲكﮑﮐﮏﮎﻜﻛﻚﻙىﻯيہەھﻰ-ﻴ\(\)«»' + isolated_char + 'ً-ِْٰٔٔأإؤئءةٓ])+', '', wikitext)
    wikitext = re.sub(r"[\s]{2,}", ' ', wikitext)
    Erab_words_List = re.findall(r'(?=\S*[أإؤئءةًٌٍَُِْٰٓٓ])\S+\s', wikitext)
    if Erab_words_List:
        for Eword in Erab_words_List:
            Eword = Eword.strip()
            without_erab_word = re.sub(r"([ًٌٍَُِّْٰٓ])+", '', Eword)
            without_erab_word2 = re.sub(r"([أإؤئءةٓ])+", '', without_erab_word)
            ErabNum = len(Eword) - len(without_erab_word2.strip())
            if ErabNum > 2 and (without_erab_word not in Erab_words):
                Erab_words.append(without_erab_word)
    wikitext = re.sub(r"([ً-ِْٰٔٔ])+", '', wikitext)
    clean_fa_text = wikitext
    wikitext = wikitext.replace('»', ' ').replace('«', ' ')
    wikitext = re.sub(r"[\(\)]", ' ', wikitext)
    wikitext = re.sub(r"[\s]{2,}", ' ', wikitext)
    wikitext = re.sub(r'(\s)' + remove_regex + r'(\s)', ' ', ' ' + wikitext + ' ')
    txt_list = wikitext.strip().split(' ')
    txt_list2 = txt_list
    txt_list = list(set(txt_list))

    faTitle2 = re.sub(r"([\(\)\^\%\$\#\@\&\,\=\{\[\}\]\'\|۱۲۳۴۵۶۷۸۹۰\?\.\!؟»«،\:؛\"\/\\\t\'\*\+\–\-\n0-9٬٫a-zA-Z\_\ـ])+", ' ', faTitle)
    faTitle_list = faTitle2.strip() + ' ' + faTitle.replace('‌', '') + ' ' + faTitle.replace('‌', ' ') + ' ' + faTitle + '‌ها' + ' ' + faTitle + '‌های' + ' ' + faTitle + '‌هایی' + ' ' + faTitle + 'ی' + ' '
    faTitle_list = faTitle_list.strip().split(' ')
    txt_list = [x for x in txt_list if x not in Erab_words]
    txt_list = [x for x in txt_list if x not in faTitle_list]
    return txt_list, clean_fa_text, Syntaxs_list, Syntaxs_list2, Erab_words, giumeh_place, txt_list2


def regex_maker(list, Correct, faText, correct_dict, my_suggestions, faTitle):
    result = []
    suggestions = []
    for wrong_word in list:
        if correct_dict:
            suggestions = [list[wrong_word]]

        if not correct_dict and my_suggestions:
            suggestions = my_suggestions.get(wrong_word, [])
        if (' ' + wrong_word + ' ' in faText) and (' ' + wrong_word + ' ' not in ' ' + faTitle + ' '):
            clean_wrong_word = remove_affixes(wrong_word)
            if '|' in clean_wrong_word:
                clean_wrong_word = clean_wrong_word.split('|')[1]
            result.append({
                "type": Correct,
                "word": wrong_word,
                "cleaned_word": clean_wrong_word,
                "suggestions": suggestions
            })
    return result


def regex_maker_slang(regex_list, Correct, faText):
    result = []
    hun_list = []
    slang_ok = []
    for regex in regex_list:
        regex = re.compile(regex, re.MULTILINE | re.UNICODE)
        slang_results = re.findall(regex, faText)
        if slang_results:
            for slangs in slang_results:
                if slangs:
                    hun_list = []
                    try:
                        slangs = slangs.strip()
                    except Exception:
                        slangs = slangs[0]
                    if slangs in slang_ok:
                        continue
                    try:
                        if not suggest.valid(slangs):
                            hun_list = suggest.suggest(slangs)
                    except Exception:
                        # print('some problems in hunspell')
                        pass
                    if ' ' + slangs + ' ' in faText:
                        clean_wrong_word = remove_affixes(slangs)
                        if '|' in clean_wrong_word:
                            clean_wrong_word = clean_wrong_word.split('|')[1]
                        slang_ok.append(slangs)
                        result.append({
                            "type": Correct,
                            "word": slangs,
                            "cleaned_word": clean_wrong_word,
                            "suggestions": hun_list
                        })
    return result, slang_ok


def remove_affixes(word: str) -> str:
    """Removes certain affixes from a word

    Args:
        word (str): A word

    Returns:
        str: The word after removing its affixes
    """
    title = None

    # remove pluralization + possessive suffixes
    stem = re.sub(r'‌?های(م|ت|ش|مان|تان|شان)$', '', word)

    # remove word-linking particles
    stem = re.sub(r'‌?های?ی?$', '', stem)

    # remove superlative-making suffixes
    stem = re.sub(r'‌?تر(ها|ین‌?ها|ین|)(ی|)$', '', stem)

    # remove attached auxiliary verbs
    stem = re.sub(r'‌?(است|بود|شد)$', '', stem)

    # remove a certain placement suffixes
    stem = re.sub(r'‌?(گذاری|گاه)$', '', stem)

    # remove singular possessive affixes
    stem = re.sub(r'‌?(ای|ام|ات|اید|اش|مان|تان|شان)$', '', stem)

    # remove partial affixes only if an affix wasn't removed already
    if stem == word:
        stem = re.sub(r'‌?(یی|ست|ند)$', '', stem)

    # remove attached affixes only if an affix wasn't removed already
    if stem == word:
        title = stem
        stem = re.sub(r'(م|ت|ش|ان|ی|یم|ید)$', '', stem)

    # remove certain prefixes
    if stem == word:
        stem = re.sub(r'^(نا|غیر|بی|با|در|بر|پر)‌?', '', stem)

    stem = stem.strip()

    if title and title != stem:
        stem = stem + "|" + stem.strip()

    return stem


def check_grammar(faText_list, words_text_list):
    past_verbs = read_data("verbs_past.txt")
    present_verbs = read_data("verbs_present.txt")

    first_step_words = []
    for item in faText_list:
        if item in words_text_list:
            continue
        item2 = re.sub(r'(ب|ن|م|می|نمی|)‌?' + past_verbs + '(ه|)‌?(بوده?|شده?|می‌شود?|)(م|ی|یم|ید|ند|ام|ای|است|ایم|اید|اند|)', '', item)
        if not item2.replace('‌', '').strip():
            continue
        item2 = re.sub(r'(ب|ن|م|می|نمی|)‌?' + present_verbs + r'(م|ی|یم|ید|ند|)', '', item)
        if not item2.replace('‌', '').strip():
            continue

        item1 = remove_affixes(item).split('|')[0]
        if item != item1:
            if item1 in words_text_list:
                continue
            if item1 + 'ن' in words_text_list:
                continue
        item1 = item.replace('‌', '').strip()
        if item != item1:
            if item1 in words_text_list:
                continue
        first_step_words.append(item)
    return first_step_words


def main(page_title, wikitext, html):
    if (
        "نظرخواهی" in page_title
        or "قهوه‌خانه" in page_title  # noqa:W503# noqa:W503
        or "تابلو" in page_title  # noqa:W503
        or "/" in page_title  # noqa:W503
    ) and "ویکی‌پدیا:" in page_title:
        test_pages = [
            "ویکی‌پدیا:اشتباه‌یاب/تست",
            "ویکی‌پدیا:اشتباه‌یاب/تمرین"
        ]
        if page_title not in test_pages:
            return {"result": [], "error": "not supposed to work on RfCs"}

    faText = (
        wikitext.replace("[[ ", "[[")
        .replace("[[ ", "[[")
        .replace(" ]]", "]]")
        .replace(" ]]", "]]")
    )
    faText = faText.replace(" |", "|").replace(" |", "|")
    if not faText.strip():
        return {"result": [], "error": "the page couldn't be retrieved"}
    result = []
    remove_regex = (
        "(" + "|".join(honorifics_blacklist) + "|" + "|".join(user_provided_blacklist) + ")"
    )
    (
        faText_list,
        clean_fa_text,
        Syntaxs_list,
        Syntaxs_list2,
        Erab_words,
        giumeh_place,
        txt_list2,
    ) = clean_text(faText, remove_regex, page_title)
    faNewText = " " + " ".join(faText_list) + " "
    clean_fa_text2 = re.sub(r"«[^»]+»", " ", clean_fa_text)
    result = result + sorted(
        regex_maker(honorifics_blacklist, 0, clean_fa_text2, False, {}, page_title),
        key=itemgetter("word"),
    )
    result_slang, slang_ok = regex_maker_slang(slang_list, 5, clean_fa_text2)
    result = result + sorted(result_slang, key=itemgetter("word"))
    result_bad, bad_ok = regex_maker_slang(profanity_pat, 6, clean_fa_text2)
    result = result + sorted(result_bad, key=itemgetter("word"))
    clean_fa_text = clean_fa_text.replace("»", " ").replace("«", " ")
    result = result + sorted(
        regex_maker(user_provided_blacklist, 2, clean_fa_text, True, {}, page_title),
        key=itemgetter("word"),
    )
    del clean_fa_text

    # --------first step check --------
    first_step_words = check_grammar(faText_list, most_words_list)
    del faText_list
    # --------Second step check --------
    second_step_words = check_grammar(first_step_words, Persian_words_list)
    del first_step_words
    # --------Third step check --------

    Third_step_words = []
    for item in second_step_words:
        if item not in wiki_titles_list:
            Third_step_words.append(item)
    del second_step_words
    for Syntaxs in Syntaxs_list:
        result = result + [{"type": 3, "word": Syntaxs, "suggestions": []}]
    # for Syntaxs in Syntaxs_list2:
    if giumeh_place:
        for Syntaxs in giumeh_place[0]:
            result = result + [{"type": 7, "word": Syntaxs, "suggestions": []}]
    hun_suggest = {}
    isolated_char = "ﭖﭗﭘﭙﭺﭻﭼﭽﮊﮋﮎﮏﮐﮑﻙﻚﻛﻜﮒﮓﮔﮕﮤﮥﯼﯽﯾﯿﻯﻰﻱﻲﻳﻴﺁﺁﺂﺄﺃﺃﺅﺅﺆﺇﺈﺇﺉﺊﺋﺌﺍﺎﺏﺐﺑﺒﺕﺖﺗﺘﺙﺚﺛﺜﺝﺞﺟﺠﺡﺢﺣﺤﺥﺦﺧﺨﺩﺪﺫﺬﺭﺮﺯﺰﺱﺲﺳﺴﺵﺶﺷﺸﺹﺺﺻﺼﺽﺾﺿﻀﻁﻂﻃﻄﻅﻆﻇﻈﻉﻊﻋﻌﻍﻎﻏﻐﻑﻒﻓﻔﻕﻖﻗﻘﻝﻞﻟﻠﻡﻢﻣﻤﻥﻦﻧﻨﻩﻪﻫﻬﻫﻭﻮﻰﻲﻵﻶﻸﻷﻹﻺﻻﻼ"
    similar_char = "كﮑﮐﮏﮎﻜﻛﻚﻙىﻯيہەھﻰ"
    Fourth_step_words = Third_step_words
    Fifth_step_words = []
    for item in Third_step_words:
        if item != re.sub(
            r"[" + isolated_char + similar_char + r"\u200e\u200f]", "", item
        ):
            Fifth_step_words.append(item)
            Fourth_step_words.remove(item)
    result = result + sorted(
        regex_maker(Fifth_step_words, 4, faNewText, False, {}, page_title),
        key=itemgetter("word"),
    )
    Fourth_step_words = [x for x in Fourth_step_words if x not in slang_ok]
    Fourth_step_words = [x for x in Fourth_step_words if x not in bad_ok]

    Six_step_words = []
    for aitem in Fourth_step_words:
        if not aitem.strip():
            continue
        if txt_list2.count(aitem) > 4:
            continue
        if "‌" in aitem:  # if the word contains ZWNJ
            if ("‌‌" not in aitem) and ("‌" not in [aitem[0], aitem[-1]]):
                aitemlist = aitem.split("‌")
                if check_grammar(aitemlist, most_words_list):
                    if not check_grammar(aitemlist, Persian_words_list):
                        continue
                else:
                    continue
        Six_step_words.append(aitem)
    Fourth_step_words = Six_step_words
    del Six_step_words

    for wo in Fourth_step_words:
        if not suggest.valid(wo):
            hun_suggest[wo] = suggest.suggest(wo)
            conj = suggest_spaced_conjunction(wo)
            if conj:
                hun_suggest[wo].append(conj)
        else:
            Fourth_step_words.remove(wo)

    # if a wrong word is repaeted at the article more than 2 times

    result = result + process_ambiguous_links(html)
    result = result + sorted(
        regex_maker(Fourth_step_words, 1, faNewText, False, hun_suggest, page_title),
        key=itemgetter("word"),
    )
    return result


def generate_output(result: list[str]) -> str:
    """Converts the result object into JSON output

    Args:
        result (list[str]): List object of the issues to return

    Returns:
        str: JSON output
    """
    return {
        "result": result,
        "types": {
            "0": {
                "color": "#9191ff",
                "title": "القاب ممنوع",
                "autofix": True,
                "syntax": False,
            },
            "1": {
                "color": "#ffc891",
                "title": "اشتباه تایپی",
                "autofix": True,
                "syntax": False,
            },
            "2": {
                "color": "#ff9191",
                "title": "غلط املائی",
                "autofix": True,
                "syntax": False,
            },
            "3": {
                "color": "#ff00e7",
                "title": "ویکی‌کد اشتباه",
                "autofix": False,
                "syntax": True,
            },
            "4": {
                "color": "#68ff00",
                "title": "نویسهٔ غیراستاندارد (نیازمند ابرابزار)",
                "autofix": True,
                "syntax": False,
            },
            "5": {
                "color": "#fff300",
                "title": "عبارت غیررسمی",
                "autofix": True,
                "syntax": False,
            },
            "6": {
                "color": "#a4a4a4",
                "title": "مشکوک به فحاشی",
                "autofix": False,
                "syntax": False,
            },
            "7": {
                "color": "#bafce9",
                "title": "نویسهٔ « یا » ناموجود",
                "autofix": False,
                "syntax": True,
            },
            "8": {
                "color": "#fadbd8",
                "title": "پیوند ابهام‌دار",
                "autofix": "D",
                "syntax": False,
            },
        },
        "version": "1.0.1",
    }


def run(page_title: str) -> str:
    """Proofreads a Wikipedia page and returns possible issues

    Args:
        page_title (str): Title of the Wikipedia page

    Returns:
        str: Output of the proofreading in JSON format
    """
    # TODO: update the JS side so it doesn't reference "Botupdate" anymore
    load_ref_data()

    page_title = page_title.replace("_", " ")
    wikitext = "\n" + get_page_wikitext(page_title) + "\n"
    html = get_page_html(page_title)
    result = main(page_title, wikitext, html)
    output = generate_output(result)

    return json.dumps(output, ensure_ascii=False)


if __name__ == "__main__":
    print(run(sys.argv[1]))