Skip to content

Commit aa2c425

Browse files
clipperhouseclipperhouse
authored
Tighten and broader VS16 handling (#24)
1 parent a9edf62 commit aa2c425

12 files changed

Lines changed: 2135 additions & 909 deletions

File tree

‎.github/workflows/gofuzz.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
runs-on: ubuntu-latest
1212
strategy:
1313
matrix:
14-
fuzzer: [FuzzBytesAndString, FuzzRune, FuzzTruncateStringAndBytes, FuzzControlSequences]
14+
fuzzer: [FuzzBytesAndString, FuzzRune, FuzzTruncateStringAndBytes, FuzzControlSequences, FuzzHasEligibleVS16Pair]
1515
steps:
1616
- name: Check out code
1717
uses: actions/checkout@v6

‎README.md‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -152,39 +152,39 @@ goarch: arm64
152152
pkg: github.com/clipperhouse/displaywidth/comparison
153153
cpu: Apple M2
154154
155-
BenchmarkString_Mixed/clipperhouse/displaywidth-8 6326 ns/op 266.66 MB/s 0 B/op 0 allocs/op
156-
BenchmarkString_Mixed/mattn/go-runewidth-8 9984 ns/op 168.97 MB/s 0 B/op 0 allocs/op
157-
BenchmarkString_Mixed/rivo/uniseg-8 19602 ns/op 86.06 MB/s 0 B/op 0 allocs/op
155+
BenchmarkString_Mixed/clipperhouse/displaywidth-8 6085 ns/op 277.23 MB/s 0 B/op 0 allocs/op
156+
BenchmarkString_Mixed/mattn/go-runewidth-8 9970 ns/op 169.21 MB/s 0 B/op 0 allocs/op
157+
BenchmarkString_Mixed/rivo/uniseg-8 19060 ns/op 88.51 MB/s 0 B/op 0 allocs/op
158158
159-
BenchmarkString_EastAsian/clipperhouse/displaywidth-8 6167 ns/op 273.55 MB/s 0 B/op 0 allocs/op
160-
BenchmarkString_EastAsian/mattn/go-runewidth-8 14022 ns/op 120.31 MB/s 0 B/op 0 allocs/op
161-
BenchmarkString_EastAsian/rivo/uniseg-8 19608 ns/op 86.04 MB/s 0 B/op 0 allocs/op
159+
BenchmarkString_EastAsian/clipperhouse/displaywidth-8 6118 ns/op 275.76 MB/s 0 B/op 0 allocs/op
160+
BenchmarkString_EastAsian/mattn/go-runewidth-8 13917 ns/op 121.22 MB/s 0 B/op 0 allocs/op
161+
BenchmarkString_EastAsian/rivo/uniseg-8 19263 ns/op 87.58 MB/s 0 B/op 0 allocs/op
162162
163-
BenchmarkString_ASCII/clipperhouse/displaywidth-8 60.62 ns/op 2111.60 MB/s 0 B/op 0 allocs/op
164-
BenchmarkString_ASCII/mattn/go-runewidth-8 122.3 ns/op 1047.01 MB/s 0 B/op 0 allocs/op
165-
BenchmarkString_ASCII/rivo/uniseg-8 1490 ns/op 85.89 MB/s 0 B/op 0 allocs/op
163+
BenchmarkString_ASCII/clipperhouse/displaywidth-8 54.54 ns/op 2347.10 MB/s 0 B/op 0 allocs/op
164+
BenchmarkString_ASCII/mattn/go-runewidth-8 125.5 ns/op 1020.32 MB/s 0 B/op 0 allocs/op
165+
BenchmarkString_ASCII/rivo/uniseg-8 1478 ns/op 86.62 MB/s 0 B/op 0 allocs/op
166166
167-
BenchmarkString_Emoji/clipperhouse/displaywidth-8 3313 ns/op 218.51 MB/s 0 B/op 0 allocs/op
168-
BenchmarkString_Emoji/mattn/go-runewidth-8 5009 ns/op 144.55 MB/s 0 B/op 0 allocs/op
169-
BenchmarkString_Emoji/rivo/uniseg-8 6868 ns/op 105.42 MB/s 0 B/op 0 allocs/op
167+
BenchmarkString_Emoji/clipperhouse/displaywidth-8 3265 ns/op 221.74 MB/s 0 B/op 0 allocs/op
168+
BenchmarkString_Emoji/mattn/go-runewidth-8 5110 ns/op 141.69 MB/s 0 B/op 0 allocs/op
169+
BenchmarkString_Emoji/rivo/uniseg-8 7137 ns/op 101.44 MB/s 0 B/op 0 allocs/op
170170
171-
BenchmarkRune_Mixed/clipperhouse/displaywidth-8 3430 ns/op 491.90 MB/s 0 B/op 0 allocs/op
172-
BenchmarkRune_Mixed/mattn/go-runewidth-8 4833 ns/op 349.09 MB/s 0 B/op 0 allocs/op
171+
BenchmarkRune_Mixed/clipperhouse/displaywidth-8 3517 ns/op 479.72 MB/s 0 B/op 0 allocs/op
172+
BenchmarkRune_Mixed/mattn/go-runewidth-8 4746 ns/op 355.48 MB/s 0 B/op 0 allocs/op
173173
174-
BenchmarkRune_EastAsian/clipperhouse/displaywidth-8 3494 ns/op 482.77 MB/s 0 B/op 0 allocs/op
175-
BenchmarkRune_EastAsian/mattn/go-runewidth-8 11724 ns/op 143.89 MB/s 0 B/op 0 allocs/op
174+
BenchmarkRune_EastAsian/clipperhouse/displaywidth-8 3454 ns/op 488.36 MB/s 0 B/op 0 allocs/op
175+
BenchmarkRune_EastAsian/mattn/go-runewidth-8 11432 ns/op 147.56 MB/s 0 B/op 0 allocs/op
176176
177-
BenchmarkRune_ASCII/clipperhouse/displaywidth-8 256.0 ns/op 500.02 MB/s 0 B/op 0 allocs/op
178-
BenchmarkRune_ASCII/mattn/go-runewidth-8 265.0 ns/op 483.01 MB/s 0 B/op 0 allocs/op
177+
BenchmarkRune_ASCII/clipperhouse/displaywidth-8 255.5 ns/op 500.88 MB/s 0 B/op 0 allocs/op
178+
BenchmarkRune_ASCII/mattn/go-runewidth-8 264.7 ns/op 483.48 MB/s 0 B/op 0 allocs/op
179179
180-
BenchmarkRune_Emoji/clipperhouse/displaywidth-8 1381 ns/op 524.30 MB/s 0 B/op 0 allocs/op
181-
BenchmarkRune_Emoji/mattn/go-runewidth-8 2345 ns/op 308.70 MB/s 0 B/op 0 allocs/op
180+
BenchmarkRune_Emoji/clipperhouse/displaywidth-8 1320 ns/op 548.44 MB/s 0 B/op 0 allocs/op
181+
BenchmarkRune_Emoji/mattn/go-runewidth-8 2286 ns/op 316.72 MB/s 0 B/op 0 allocs/op
182182
183-
BenchmarkTruncateWithTail/clipperhouse/displaywidth-8 2755 ns/op 64.24 MB/s 192 B/op 14 allocs/op
184-
BenchmarkTruncateWithTail/mattn/go-runewidth-8 4683 ns/op 37.80 MB/s 192 B/op 14 allocs/op
183+
BenchmarkTruncateWithTail/clipperhouse/displaywidth-8 2495 ns/op 70.94 MB/s 192 B/op 14 allocs/op
184+
BenchmarkTruncateWithTail/mattn/go-runewidth-8 4569 ns/op 38.74 MB/s 192 B/op 14 allocs/op
185185
186-
BenchmarkTruncateWithoutTail/clipperhouse/displaywidth-8 2481 ns/op 92.30 MB/s 0 B/op 0 allocs/op
187-
BenchmarkTruncateWithoutTail/mattn/go-runewidth-8 5334 ns/op 42.93 MB/s 0 B/op 0 allocs/op
186+
BenchmarkTruncateWithoutTail/clipperhouse/displaywidth-8 2456 ns/op 93.25 MB/s 0 B/op 0 allocs/op
187+
BenchmarkTruncateWithoutTail/mattn/go-runewidth-8 5182 ns/op 44.19 MB/s 0 B/op 0 allocs/op
188188
```
189189

190190
Here are some notes on [how to make Unicode things fast](https://clipperhouse.com/go-unicode/).

‎comparison/README.md‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,37 +19,37 @@ goarch: arm64
1919
pkg: github.com/clipperhouse/displaywidth/comparison
2020
cpu: Apple M2
2121
22-
BenchmarkString_Mixed/clipperhouse/displaywidth-8 6326 ns/op 266.66 MB/s 0 B/op 0 allocs/op
23-
BenchmarkString_Mixed/mattn/go-runewidth-8 9984 ns/op 168.97 MB/s 0 B/op 0 allocs/op
24-
BenchmarkString_Mixed/rivo/uniseg-8 19602 ns/op 86.06 MB/s 0 B/op 0 allocs/op
22+
BenchmarkString_Mixed/clipperhouse/displaywidth-8 6085 ns/op 277.23 MB/s 0 B/op 0 allocs/op
23+
BenchmarkString_Mixed/mattn/go-runewidth-8 9970 ns/op 169.21 MB/s 0 B/op 0 allocs/op
24+
BenchmarkString_Mixed/rivo/uniseg-8 19060 ns/op 88.51 MB/s 0 B/op 0 allocs/op
2525
26-
BenchmarkString_EastAsian/clipperhouse/displaywidth-8 6167 ns/op 273.55 MB/s 0 B/op 0 allocs/op
27-
BenchmarkString_EastAsian/mattn/go-runewidth-8 14022 ns/op 120.31 MB/s 0 B/op 0 allocs/op
28-
BenchmarkString_EastAsian/rivo/uniseg-8 19608 ns/op 86.04 MB/s 0 B/op 0 allocs/op
26+
BenchmarkString_EastAsian/clipperhouse/displaywidth-8 6118 ns/op 275.76 MB/s 0 B/op 0 allocs/op
27+
BenchmarkString_EastAsian/mattn/go-runewidth-8 13917 ns/op 121.22 MB/s 0 B/op 0 allocs/op
28+
BenchmarkString_EastAsian/rivo/uniseg-8 19263 ns/op 87.58 MB/s 0 B/op 0 allocs/op
2929
30-
BenchmarkString_ASCII/clipperhouse/displaywidth-8 60.62 ns/op 2111.60 MB/s 0 B/op 0 allocs/op
31-
BenchmarkString_ASCII/mattn/go-runewidth-8 122.3 ns/op 1047.01 MB/s 0 B/op 0 allocs/op
32-
BenchmarkString_ASCII/rivo/uniseg-8 1490 ns/op 85.89 MB/s 0 B/op 0 allocs/op
30+
BenchmarkString_ASCII/clipperhouse/displaywidth-8 54.54 ns/op 2347.10 MB/s 0 B/op 0 allocs/op
31+
BenchmarkString_ASCII/mattn/go-runewidth-8 125.5 ns/op 1020.32 MB/s 0 B/op 0 allocs/op
32+
BenchmarkString_ASCII/rivo/uniseg-8 1478 ns/op 86.62 MB/s 0 B/op 0 allocs/op
3333
34-
BenchmarkString_Emoji/clipperhouse/displaywidth-8 3313 ns/op 218.51 MB/s 0 B/op 0 allocs/op
35-
BenchmarkString_Emoji/mattn/go-runewidth-8 5009 ns/op 144.55 MB/s 0 B/op 0 allocs/op
36-
BenchmarkString_Emoji/rivo/uniseg-8 6868 ns/op 105.42 MB/s 0 B/op 0 allocs/op
34+
BenchmarkString_Emoji/clipperhouse/displaywidth-8 3265 ns/op 221.74 MB/s 0 B/op 0 allocs/op
35+
BenchmarkString_Emoji/mattn/go-runewidth-8 5110 ns/op 141.69 MB/s 0 B/op 0 allocs/op
36+
BenchmarkString_Emoji/rivo/uniseg-8 7137 ns/op 101.44 MB/s 0 B/op 0 allocs/op
3737
38-
BenchmarkRune_Mixed/clipperhouse/displaywidth-8 3430 ns/op 491.90 MB/s 0 B/op 0 allocs/op
39-
BenchmarkRune_Mixed/mattn/go-runewidth-8 4833 ns/op 349.09 MB/s 0 B/op 0 allocs/op
38+
BenchmarkRune_Mixed/clipperhouse/displaywidth-8 3517 ns/op 479.72 MB/s 0 B/op 0 allocs/op
39+
BenchmarkRune_Mixed/mattn/go-runewidth-8 4746 ns/op 355.48 MB/s 0 B/op 0 allocs/op
4040
41-
BenchmarkRune_EastAsian/clipperhouse/displaywidth-8 3494 ns/op 482.77 MB/s 0 B/op 0 allocs/op
42-
BenchmarkRune_EastAsian/mattn/go-runewidth-8 11724 ns/op 143.89 MB/s 0 B/op 0 allocs/op
41+
BenchmarkRune_EastAsian/clipperhouse/displaywidth-8 3454 ns/op 488.36 MB/s 0 B/op 0 allocs/op
42+
BenchmarkRune_EastAsian/mattn/go-runewidth-8 11432 ns/op 147.56 MB/s 0 B/op 0 allocs/op
4343
44-
BenchmarkRune_ASCII/clipperhouse/displaywidth-8 256.0 ns/op 500.02 MB/s 0 B/op 0 allocs/op
45-
BenchmarkRune_ASCII/mattn/go-runewidth-8 265.0 ns/op 483.01 MB/s 0 B/op 0 allocs/op
44+
BenchmarkRune_ASCII/clipperhouse/displaywidth-8 255.5 ns/op 500.88 MB/s 0 B/op 0 allocs/op
45+
BenchmarkRune_ASCII/mattn/go-runewidth-8 264.7 ns/op 483.48 MB/s 0 B/op 0 allocs/op
4646
47-
BenchmarkRune_Emoji/clipperhouse/displaywidth-8 1381 ns/op 524.30 MB/s 0 B/op 0 allocs/op
48-
BenchmarkRune_Emoji/mattn/go-runewidth-8 2345 ns/op 308.70 MB/s 0 B/op 0 allocs/op
47+
BenchmarkRune_Emoji/clipperhouse/displaywidth-8 1320 ns/op 548.44 MB/s 0 B/op 0 allocs/op
48+
BenchmarkRune_Emoji/mattn/go-runewidth-8 2286 ns/op 316.72 MB/s 0 B/op 0 allocs/op
4949
50-
BenchmarkTruncateWithTail/clipperhouse/displaywidth-8 2755 ns/op 64.24 MB/s 192 B/op 14 allocs/op
51-
BenchmarkTruncateWithTail/mattn/go-runewidth-8 4683 ns/op 37.80 MB/s 192 B/op 14 allocs/op
50+
BenchmarkTruncateWithTail/clipperhouse/displaywidth-8 2495 ns/op 70.94 MB/s 192 B/op 14 allocs/op
51+
BenchmarkTruncateWithTail/mattn/go-runewidth-8 4569 ns/op 38.74 MB/s 192 B/op 14 allocs/op
5252
53-
BenchmarkTruncateWithoutTail/clipperhouse/displaywidth-8 2481 ns/op 92.30 MB/s 0 B/op 0 allocs/op
54-
BenchmarkTruncateWithoutTail/mattn/go-runewidth-8 5334 ns/op 42.93 MB/s 0 B/op 0 allocs/op
53+
BenchmarkTruncateWithoutTail/clipperhouse/displaywidth-8 2456 ns/op 93.25 MB/s 0 B/op 0 allocs/op
54+
BenchmarkTruncateWithoutTail/mattn/go-runewidth-8 5182 ns/op 44.19 MB/s 0 B/op 0 allocs/op
5555
```

‎fuzz_test.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,3 +443,77 @@ func FuzzControlSequences(f *testing.F) {
443443
}
444444
})
445445
}
446+
447+
// FuzzHasEligibleVS16Pair fuzzes byte-level VS16 detection against
448+
// a slower UTF-8-decoding reference implementation.
449+
func FuzzHasEligibleVS16Pair(f *testing.F) {
450+
if testing.Short() {
451+
f.Skip("skipping fuzz test in short mode")
452+
}
453+
454+
seeds := []string{
455+
"",
456+
"a",
457+
"a\uFE0F", // invalid VS16 base
458+
"✡\uFE0F", // valid immediate VS16 pair
459+
"👩‍❤️‍👨", // later VS16 in ZWJ sequence
460+
"\u26F9\U0001F3FB\u200D\u2642\uFE0F", // later VS16 in skin-tone+gender sequence
461+
"\u26F9\u0301\uFE0E\u200D\u2660\uFE0F", // non-FE0F 0xEF (FE0E) before the real FE0F
462+
"\u26F9\uFE20\u200D\u2660\uFE0F", // non-FE0F 0xEF (FE20) before the real FE0F
463+
"\xff\xfe\xfd", // invalid UTF-8
464+
}
465+
for _, s := range seeds {
466+
f.Add([]byte(s), uint16(0))
467+
f.Add([]byte(s), uint16(1))
468+
f.Add([]byte(s), uint16(7))
469+
}
470+
471+
f.Fuzz(func(t *testing.T, text []byte, startSeed uint16) {
472+
// Exercise a range that includes in-bounds and out-of-bounds starts.
473+
start := int(startSeed)
474+
if len(text) > 0 {
475+
start %= (len(text) + 3)
476+
}
477+
478+
gotBytes := hasEligibleVS16Pair(text, start)
479+
gotString := hasEligibleVS16Pair(string(text), start)
480+
if gotBytes != gotString {
481+
t.Errorf("hasEligibleVS16Pair bytes/string mismatch for %q start=%d: %v != %v", text, start, gotBytes, gotString)
482+
}
483+
484+
want := hasEligibleVS16PairReference(text, start)
485+
if gotBytes != want {
486+
t.Errorf("hasEligibleVS16Pair(%q, start=%d) = %v, want %v", text, start, gotBytes, want)
487+
}
488+
})
489+
}
490+
491+
func hasEligibleVS16PairReference(b []byte, start int) bool {
492+
if len(b) == 0 || start >= len(b) {
493+
return false
494+
}
495+
if start < 0 {
496+
start = 0
497+
}
498+
499+
i := 0
500+
prevStart := -1
501+
for i < len(b) {
502+
r, sz := utf8.DecodeRune(b[i:])
503+
if sz <= 0 {
504+
break
505+
}
506+
507+
if i >= start && r == '\uFE0F' && prevStart >= 0 {
508+
p, rsz := lookup(b[prevStart:])
509+
if rsz > 0 && prevStart+rsz == i && property(p).is(_VS16_Eligible) {
510+
return true
511+
}
512+
}
513+
514+
prevStart = i
515+
i += sz
516+
}
517+
518+
return false
519+
}

‎graphemes.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
//
99
// Iterate using the Next method, and get the width of the current grapheme
1010
// using the Width method.
11-
type Graphemes[T ~string | []byte] struct {
11+
type Graphemes[T ~string | ~[]byte] struct {
1212
iter *graphemes.Iterator[T]
1313
options Options
1414
}

0 commit comments

Comments
 (0)