Helti-bot/bot_utils.py at master · TeamPraxidike/Helti-bot · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
"""
import classla
#classla.download('bg')
nlp = classla.Pipeline('bg') # run classla.download('bg') beforehand if necessary
doc = nlp("Алеко Константинов е роден в Свищов.")
print(doc)

"""

"""
basic demo script
"""

import sys
import argparse
import os

import classla
from classla.resources.common import DEFAULT_MODEL_DIR
from classla.utils.conll import CoNLL


def StripOfChar(listOfLemmas):
    '''
    ~ Removes any unnecessary characters after lemmatization
    '''
    for x in listOfLemmas:
        if "-" not in x:
            continue
        else:
            for y in range(len(x)):
                if x[y] != "-":
                    continue
                else:
                    listOfLemmas[listOfLemmas.index(x)] = x.replace(x[y:len(x)],"")
                    break

    return listOfLemmas


"""
if __name__ == '__main__':
    # get arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--models_dir', help='location of models files | default: ~/classla_resources',
                        default=DEFAULT_MODEL_DIR)
    parser.add_argument('-l', '--lang', help='Demo language',
                        default="bg")
    parser.add_argument('-c', '--cpu', action='store_true', help='Use cpu as the device.')
    args = parser.parse_args()

    example_sentences = {"bg": ["Алеко Константинов беше роден в Свищов, но аз съща съм се родил там.", "Имало едно време един голям карък"]}

    if args.lang not in example_sentences:
        print(f'Sorry, but we don\'t have a demo sentence for "{args.lang}" for the moment. Try one of these languages: {list(example_sentences.keys())}')
        sys.exit(1)

    # download the models
    classla.download(args.lang, args.models_dir)


    # set up a pipeline
    print('---')
    print('Building pipeline...')
    pipeline = classla.Pipeline(dir=args.models_dir, lang=args.lang, use_gpu=(not args.cpu))

    print((example_sentences[args.lang]))
    # process the document
    doc = pipeline(example_sentences[args.lang][0])


    # access nlp annotations
    print('')
    print('Input: {}'.format(example_sentences[args.lang]))
    print("The tokenizer split the input into {} sentences.".format(len(doc.sentences)))
    print('---')

    print('tokens of first sentence: ')
    strang = doc.sentences[0].tokens_string()
    print(strang)
    print('')
    print('---')

    print('dependency parse of first sentence: ')
    doc.sentences[0].print_dependencies()
    print('')
    print('---')

    '''
    -> sent in doc.sentences returns a 2d list
    -> word in sent.words returns keys from dictionary saved in the first list
    -> the next operation returns the value from the said key
    ---> in this case the k:v pair is 'lemma':'value'
    '''

    lemmas = [f'{word.lemma}' for sent in doc.sentences for word in sent.words]

    cleanLemmas = StripOfChar(lemmas)
    print(cleanLemmas)
    print('')
    print('---')
"""