Skip to content

Commit 9a03a47

Browse files
Ken KundertKen Kundert
Ken Kundert
authored and
Ken Kundert
committed
fix feature recognition in proposed tests
1 parent defd3a3 commit 9a03a47

3 files changed

Lines changed: 69 additions & 36 deletions

File tree

‎proposed_tests/convert‎

Lines changed: 43 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
11
#!/usr/bin/env python3
2+
"""
3+
Convert
4+
5+
usage:
6+
convert [<tests.nt>]
7+
8+
Converts the human-friendly test definitions file (default is tests.nt) to the
9+
machine-friendly tests file (default is tests.json). Running this command is
10+
only necessary if you change the test definitions file.
11+
"""
12+
213

314
# IMPORTS {{{1
415
import nestedtext as nt
5-
from inform import (
6-
Error, fatal, indent, is_str, is_mapping, is_collection, os_error,
7-
terminate, warn
8-
)
9-
from base64 import b64encode
10-
import codecs
1116
try:
1217
from nestedtext.nestedtext import Lines
1318
except ImportError:
@@ -20,12 +25,21 @@ except ImportError:
2025
"line types are not available."
2126
)
2227
Lines = None
28+
from base64 import b64encode, b64decode
2329
from collections import defaultdict
30+
from docopt import docopt
31+
from inform import (
32+
Error, fatal, indent, is_str, is_mapping, is_collection, os_error,
33+
terminate, warn
34+
)
2435
from itertools import batched
36+
from pathlib import Path
2537
from voluptuous import (
2638
Schema, Optional, Required, Any, Self, Invalid, MultipleInvalid
2739
)
2840
from voluptuous_errors import report_voluptuous_errors
41+
import codecs
42+
import io
2943
import re
3044
import json
3145

@@ -84,9 +98,15 @@ def fix_eol(match):
8498

8599
# extract line types {{{1
86100
if Lines:
87-
def extract_line_types(text):
101+
def extract_line_types(content):
102+
try:
103+
content = b64decode(content).decode('utf-8-sig', errors='strict')
104+
except UnicodeError as e:
105+
return dict(unrecognized = 1)
106+
88107
types = defaultdict(int)
89-
lines = Lines(text.splitlines(), True)
108+
lines = Lines([], True)
109+
lines.lines = io.StringIO(content, newline=None)
90110

91111
for line in lines.read_lines():
92112
types[line.kind] += 1
@@ -119,10 +139,15 @@ def bytes_encode(given, encoding):
119139

120140

121141
# CONVERT {{{1
122-
# read tests.nt {{{1
142+
# read command line {{{1
143+
cmdline = docopt(__doc__)
144+
input_path = Path(cmdline['<tests.nt>'] or 'tests.nt')
145+
output_path = input_path.with_suffix('.json')
146+
147+
# read test definitions file (tests.nt) {{{1
123148
try:
124149
keymap = {}
125-
tests = nt.load("tests.nt", keymap=keymap)
150+
tests = nt.load(input_path, keymap=keymap)
126151
tests = tests_validator(tests)
127152
except OSError as e:
128153
fatal(os_error(e))
@@ -168,17 +193,21 @@ try:
168193
load_out = load_out,
169194
load_err = load_err,
170195
encoding = encoding,
171-
types = extract_line_types(load_in),
196+
types = extract_line_types(load_in_encoded)
172197
)
173198
for line_type, count in processed_test['types'].items():
174199
accumulated_line_types[line_type] += count
175200

201+
if 'unrecognized' in processed_test['types'] and not load_err:
202+
# raise Error("unrecognized line in a test that does not expect an error.")
203+
warn("unrecognized line in a test that does not expect an error.",
204+
culprit=key)
205+
176206
processed[key] = processed_test
177207

178208
# write tests.json {{{1
179-
with open('tests.json', 'w') as f:
180-
json.dump(dict(load_tests=processed), f, indent=4, ensure_ascii=False)
181-
f.write('\n')
209+
content = json.dumps(dict(load_tests=processed), indent=4, ensure_ascii=False)
210+
output_path.write_text(content, encoding="utf-8")
182211

183212
except OSError as e:
184213
fatal(os_error(e))

‎proposed_tests/tests.json‎

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -865,7 +865,7 @@
865865
"encoding": "utf-8",
866866
"types": {
867867
"dict item": 12,
868-
"unrecognized": 1
868+
"blank": 3
869869
}
870870
},
871871
"pelvis": {
@@ -888,7 +888,7 @@
888888
"encoding": "utf-8",
889889
"types": {
890890
"dict item": 12,
891-
"unrecognized": 1
891+
"blank": 3
892892
}
893893
},
894894
"pursuit": {
@@ -911,7 +911,7 @@
911911
"encoding": "utf-8",
912912
"types": {
913913
"dict item": 12,
914-
"unrecognized": 1
914+
"blank": 3
915915
}
916916
},
917917
"beset": {
@@ -1102,7 +1102,8 @@
11021102
"encoding": "utf-8",
11031103
"types": {
11041104
"comment": 1,
1105-
"dict item": 2
1105+
"dict item": 1,
1106+
"unrecognized": 1
11061107
}
11071108
},
11081109
"concuss": {
@@ -2770,7 +2771,8 @@
27702771
},
27712772
"encoding": "utf-8",
27722773
"types": {
2773-
"dict item": 2
2774+
"dict item": 1,
2775+
"unrecognized": 1
27742776
}
27752777
},
27762778
"tombstone": {
@@ -2795,8 +2797,7 @@
27952797
"load_err": {},
27962798
"encoding": "utf-8",
27972799
"types": {
2798-
"dict item": 1,
2799-
"unrecognized": 1
2800+
"dict item": 2
28002801
}
28012802
},
28022803
"rascal": {
@@ -2809,9 +2810,8 @@
28092810
},
28102811
"encoding": "utf-8",
28112812
"types": {
2812-
"dict item": 1,
2813-
"string item": 3,
2814-
"unrecognized": 2
2813+
"dict item": 3,
2814+
"string item": 3
28152815
}
28162816
},
28172817
"truncheon": {
@@ -2824,13 +2824,12 @@
28242824
},
28252825
"encoding": "utf-8",
28262826
"types": {
2827-
"list item": 1,
2828-
"string item": 3,
2829-
"unrecognized": 2
2827+
"list item": 3,
2828+
"string item": 3
28302829
}
28312830
},
28322831
"vagrant": {
2833-
"load_in": "aW5saW5lIGRpY3Q6CiAgICB7YTpBLCBiOiBCLGMgOkMsIGQgOiBEIH0KaW5saW5lIGxpc3Q6CiAgICBbYSwgYixjICwgZCBd",
2832+
"load_in": "aW5saW5lIGRpY3Q6CiAgICB7YTpBLCBiOiBCLGMgOkMgLCBkIDogRCB9CmlubGluZSBsaXN0OgogICAgW2EsIGIsYyAsIGQgXQ==",
28342833
"load_out": {
28352834
"inline dict": {
28362835
"a": "A",
@@ -2887,9 +2886,7 @@
28872886
},
28882887
"encoding": "utf-16",
28892888
"types": {
2890-
"dict item": 7,
2891-
"list item": 2,
2892-
"string item": 2
2889+
"unrecognized": 1
28932890
}
28942891
},
28952892
"amendment": {
@@ -2903,7 +2900,7 @@
29032900
},
29042901
"encoding": "bytes",
29052902
"types": {
2906-
"string item": 3
2903+
"unrecognized": 1
29072904
}
29082905
},
29092906
"academic": {
@@ -2912,7 +2909,7 @@
29122909
"load_err": {},
29132910
"encoding": "bytes",
29142911
"types": {
2915-
"unrecognized": 1
2912+
"inline dict": 1
29162913
}
29172914
},
29182915
"paragon": {
@@ -2960,4 +2957,4 @@
29602957
}
29612958
}
29622959
}
2963-
}
2960+
}

‎proposed_tests/tests.nt‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -735,6 +735,7 @@ subdue:
735735
> treasurer:
736736
> name: Fumiko Purvis
737737
> address: \x20
738+
# \x20 is a plain ASCII space, it is given as \x20 to make it explicit.
738739
> > 3636 Buffalo Ave
739740
> > Topeka, Kansas 20692
740741
load_err:
@@ -3126,10 +3127,11 @@ tombstone:
31263127
# cavern {{{1
31273128
cavern:
31283129
description:
3129-
> inline dictionary items must ignore whitespace following the colon
3130+
> dictionary items must ignore a space following the colon
31303131
string_in:
31313132
> key 1:
31323133
> key 2:\x20
3134+
# \x20 is a plain ASCII space, it is given as \x20 to make it explicit.
31333135
load_out:
31343136
key 1:
31353137
key 2:
@@ -3142,6 +3144,7 @@ rascal:
31423144
> key 1:
31433145
> > value 1
31443146
> key 2:\x20
3147+
# \x20 is a plain ASCII space, it is given as \x20 to make it explicit.
31453148
> > value 2
31463149
> key 3:\x20\x20
31473150
> > value 3
@@ -3158,6 +3161,7 @@ truncheon:
31583161
> -
31593162
> > value 1
31603163
> -\x20
3164+
# \x20 is a plain ASCII space, it is given as \x20 to make it explicit.
31613165
> > value 2
31623166
> -\x20\x20
31633167
> > value 3
@@ -3173,7 +3177,7 @@ vagrant:
31733177
> must trim whitespace from both sides of values
31743178
string_in:
31753179
> inline dict:
3176-
> {a:A, b: B,c :C, d : D }
3180+
> {a:A, b: B,c :C , d : D }
31773181
> inline list:
31783182
> [a, b,c , d ]
31793183
load_out:
@@ -3265,6 +3269,7 @@ academic:
32653269
load_out: !{}
32663270

32673271
# paragon {{{1
3272+
# test donated by jonjanisch
32683273
paragon:
32693274
description: invalid mix of string and list items
32703275
string_in:
@@ -3278,6 +3283,7 @@ paragon:
32783283
colno: 0
32793284

32803285
# chemist {{{1
3286+
# test donated by jonjanisch
32813287
chemist:
32823288
description: invalid mix of string and dict items
32833289
string_in:
@@ -3291,6 +3297,7 @@ chemist:
32913297
colno: 0
32923298

32933299
# neophyte {{{1
3300+
# test donated by jonjanisch
32943301
neophyte:
32953302
description: invalid indentation
32963303
string_in:

0 commit comments

Comments
 (0)