Skip to content

Commit f741e6a

Browse files
committed
fix: Handling of non-ASCII identifiers
Non-ASCII-identifiers where not handled safely: ``` >>> ö : Int thread 'main' panicked at ~/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokay-0.6.12/src/compiler/ast.rs:1169:42: byte index 1 is not a char boundary; it is inside 'ö' (bytes 0..2) of `ö` note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace ```
1 parent 3b473ba commit f741e6a

3 files changed

Lines changed: 41 additions & 15 deletions

File tree

src/compiler/ast.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ fn traverse_node_value(scope: &Scope, node: &Dict, name: Option<String>) -> ImlV
251251
else {
252252
format!(
253253
"Argument named '{}' invalid; Use a name starting in lower-case, e.g. '{}{}'",
254-
name, &name[0..1].to_lowercase(), &name[1..]
254+
name, name.chars().next().unwrap().to_lowercase(), name.chars().skip(1).collect::<String>()
255255
)
256256
}
257257
);
@@ -570,7 +570,7 @@ fn traverse_node_lvalue(scope: &Scope, node: &Dict, store: bool, hold: bool) ->
570570
Some(_) => {
571571
scope.push_error(
572572
traverse_node_offset(node),
573-
format!("Cannot assign to constant '{}'", name),
573+
format!("Cannot assign variable value to constant '{}'", name),
574574
);
575575
break 'load;
576576
}
@@ -591,7 +591,7 @@ fn traverse_node_lvalue(scope: &Scope, node: &Dict, store: bool, hold: bool) ->
591591
scope.push_error(
592592
traverse_node_offset(node),
593593

594-
if &name[0..1] == "_" {
594+
if name.chars().nth(0) == Some('_') {
595595
format!(
596596
"The variable '{}' is invalid, only constants may start with '_'",
597597
name
@@ -1165,8 +1165,8 @@ fn traverse_node(scope: &Scope, node: &Dict) -> ImlOp {
11651165
scope.push_error(
11661166
traverse_node_offset(node),
11671167
format!(
1168-
"Cannot assign to constant '{}' as consumable. Use an identifier starting in upper-case, e.g. '{}{}'",
1169-
ident, &ident[0..1].to_uppercase(), &ident[1..]
1168+
"Cannot assign consumable to constant '{}'. Use an identifier starting in upper-case, e.g. '{}{}'",
1169+
ident, ident.chars().next().unwrap().to_uppercase(), ident.chars().skip(1).collect::<String>()
11701170
)
11711171
);
11721172

@@ -1177,14 +1177,14 @@ fn traverse_node(scope: &Scope, node: &Dict) -> ImlOp {
11771177
traverse_node_offset(node),
11781178
if ident.starts_with("_") {
11791179
format!(
1180-
"Cannot assign to constant '{}', because it must be consumable. Use an identifier not starting with '_'.",
1180+
"Cannot assign non-consumable to constant '{}'. Use an identifier not starting with '_'.",
11811181
ident
11821182
)
11831183
}
11841184
else {
11851185
format!(
1186-
"Cannot assign to constant '{}', because it must be consumable. Use an identifier starting in lower-case, e.g. '{}{}'",
1187-
ident, &ident[0..1].to_lowercase(), &ident[1..]
1186+
"Cannot assign non-consumable to constant '{}'. Use an identifier starting in lower-case, e.g. '{}{}'",
1187+
ident, ident.chars().next().unwrap().to_lowercase(), ident.chars().skip(1).collect::<String>()
11881188
)
11891189
}
11901190
);

tests/compiler_identifiers.tok

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,27 @@ Factorial : factorial # Error: Cannot assign non-consumable to consumable const
1313

1414
IsOkay : @{
1515
Int if $1 > 100 && $1 < 1000 accept
16-
} # Ok, because the function is a parselet as it calls Cident
16+
} # Ok, because the function is a parselet as it calls Ints
1717

1818
_ : "abc"
1919

2020
# https://github.com/tokay-lang/tokay/issues/149
2121
a : 1
2222
a = 2
23+
Ö = 1
2324

2425
b = 3
2526
b : 4
27+
ö : Int
28+
Ä : 123
2629

2730
#---
28-
#ERR:Line 1, column 1: Cannot assign to constant 'Pi', because it must be consumable. Use an identifier starting in lower-case, e.g. 'pi'
29-
#ERR:Line 5, column 1: Cannot assign to constant 'cident' as consumable. Use an identifier starting in upper-case, e.g. 'Cident'
30-
#ERR:Line 12, column 1: Cannot assign to constant 'Factorial', because it must be consumable. Use an identifier starting in lower-case, e.g. 'factorial'
31-
#ERR:Line 18, column 1: Cannot assign to constant '_', because it must be consumable. Use an identifier not starting with '_'.
32-
#ERR:Line 22, column 1: Cannot assign to constant 'a'
33-
#ERR:Line 25, column 1: Cannot assign constant value to variable 'b'
31+
#ERR:Line 1, column 1: Cannot assign non-consumable to constant 'Pi'. Use an identifier starting in lower-case, e.g. 'pi'
32+
#ERR:Line 5, column 1: Cannot assign consumable to constant 'cident'. Use an identifier starting in upper-case, e.g. 'Cident'
33+
#ERR:Line 12, column 1: Cannot assign non-consumable to constant 'Factorial'. Use an identifier starting in lower-case, e.g. 'factorial'
34+
#ERR:Line 18, column 1: Cannot assign non-consumable to constant '_'. Use an identifier not starting with '_'.
35+
#ERR:Line 22, column 1: Cannot assign variable value to constant 'a'
36+
#ERR:Line 23, column 1: Cannot assign variable named 'Ö'; Try lower-case identifier, e.g. 'ö'
37+
#ERR:Line 26, column 1: Cannot assign constant value to variable 'b'
38+
#ERR:Line 27, column 1: Cannot assign consumable to constant 'ö'. Use an identifier starting in upper-case, e.g. 'Ö'
39+
#ERR:Line 28, column 1: Cannot assign non-consumable to constant 'Ä'. Use an identifier starting in lower-case, e.g. 'ä'

tests/str_find.tok

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#testmode:repl
2+
s = "hello"
3+
4+
s.find("l",3)
5+
s.find("l",2)
6+
s.find("l",1)
7+
s.find("l",0)
8+
s.find("l",4)
9+
s.find("l",3,4)
10+
s.find("l",3,3)
11+
s.find("l",3,2)
12+
#---
13+
#3
14+
#2
15+
#2
16+
#2
17+
#-1
18+
#3
19+
#-1
20+
#-1

0 commit comments

Comments
 (0)