mirror of
https://github.com/docker/cli.git
synced 2026-09-26 17:30:51 -04:00
Merge pull request #7293 from thaJeztah/bump_uax29
vendor: github.com/clipperhouse/uax29/v2 v2.7.0
This commit is contained in:
Paweł Gronowski
+1
-1
@@ -75,7 +75,7 @@ require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
|
||||
github.com/docker/go-events v0.1.0 // indirect
|
||||
github.com/docker/go-metrics v0.1.0 // indirect
|
||||
|
||||
+2
-2
@@ -12,8 +12,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
|
||||
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
|
||||
+63
-25
@@ -1,32 +1,34 @@
|
||||
An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode version 15.0.0.
|
||||
An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode 17.
|
||||
|
||||
[](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
|
||||
|
||||
|
||||
|
||||
## Quick start
|
||||
|
||||
```
|
||||
go get "github.com/clipperhouse/uax29/v2/graphemes"
|
||||
go get github.com/clipperhouse/uax29/v2/graphemes
|
||||
```
|
||||
|
||||
```go
|
||||
import "github.com/clipperhouse/uax29/v2/graphemes"
|
||||
|
||||
text := "Hello, 世界. Nice dog! 👍🐶"
|
||||
g := graphemes.FromString(text)
|
||||
|
||||
tokens := graphemes.FromString(text)
|
||||
|
||||
for tokens.Next() { // Next() returns true until end of data
|
||||
fmt.Println(tokens.Value()) // Do something with the current grapheme
|
||||
for g.Next() { // Next() returns true until end of data
|
||||
fmt.Println(g.Value()) // Do something with the current grapheme
|
||||
}
|
||||
```
|
||||
|
||||
[](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
|
||||
|
||||
_A grapheme is a “single visible character”, which might be a simple as a single letter, or a complex emoji that consists of several Unicode code points._
|
||||
|
||||
## Conformance
|
||||
|
||||
We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Tests29). Status:
|
||||
We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-36.html#Tests29).
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## APIs
|
||||
|
||||
@@ -34,11 +36,10 @@ We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Te
|
||||
|
||||
```go
|
||||
text := "Hello, 世界. Nice dog! 👍🐶"
|
||||
g := graphemes.FromString(text)
|
||||
|
||||
tokens := graphemes.FromString(text)
|
||||
|
||||
for tokens.Next() { // Next() returns true until end of data
|
||||
fmt.Println(tokens.Value()) // Do something with the current grapheme
|
||||
for g.Next() { // Next() returns true until end of data
|
||||
fmt.Println(g.Value()) // Do something with the current grapheme
|
||||
}
|
||||
```
|
||||
|
||||
@@ -47,15 +48,15 @@ for tokens.Next() { // Next() returns true until end of data
|
||||
`FromReader` embeds a [`bufio.Scanner`](https://pkg.go.dev/bufio#Scanner), so just use those methods.
|
||||
|
||||
```go
|
||||
r := getYourReader() // from a file or network maybe
|
||||
tokens := graphemes.FromReader(r)
|
||||
r := getYourReader() // from a file or network maybe
|
||||
g := graphemes.FromReader(r)
|
||||
|
||||
for tokens.Scan() { // Scan() returns true until error or EOF
|
||||
fmt.Println(tokens.Text()) // Do something with the current grapheme
|
||||
for g.Scan() { // Scan() returns true until error or EOF
|
||||
fmt.Println(g.Text()) // Do something with the current grapheme
|
||||
}
|
||||
|
||||
if tokens.Err() != nil { // Check the error
|
||||
log.Fatal(tokens.Err())
|
||||
if g.Err() != nil { // Check the error
|
||||
log.Fatal(g.Err())
|
||||
}
|
||||
```
|
||||
|
||||
@@ -64,16 +65,53 @@ if tokens.Err() != nil { // Check the error
|
||||
```go
|
||||
b := []byte("Hello, 世界. Nice dog! 👍🐶")
|
||||
|
||||
tokens := graphemes.FromBytes(b)
|
||||
g := graphemes.FromBytes(b)
|
||||
|
||||
for tokens.Next() { // Next() returns true until end of data
|
||||
fmt.Println(tokens.Value()) // Do something with the current grapheme
|
||||
for g.Next() { // Next() returns true until end of data
|
||||
fmt.Println(g.Value()) // Do something with the current grapheme
|
||||
}
|
||||
```
|
||||
|
||||
### Performance
|
||||
### ANSI escape sequences
|
||||
|
||||
On a Mac M2 laptop, we see around 200MB/s, or around 100 million graphemes per second. You should see ~constant memory, and no allocations.
|
||||
By the UAX 29 specification, ANSI escape sequences are not grapheme clusters. To treat 7-bit ANSI escape sequences as a single cluster, set `AnsiEscapeSequences` to true.
|
||||
|
||||
```go
|
||||
text := "Hello, \x1b[31mworld\x1b[0m!"
|
||||
g := graphemes.FromString(text)
|
||||
g.AnsiEscapeSequences = true
|
||||
|
||||
for g.Next() {
|
||||
fmt.Println(g.Value())
|
||||
}
|
||||
```
|
||||
|
||||
To also parse 8-bit C1 controls (non-UTF-8 bytes), set `AnsiEscapeSequences8Bit` to true.
|
||||
|
||||
```go
|
||||
g.AnsiEscapeSequences = true // 7-bit forms (ESC ...)
|
||||
g.AnsiEscapeSequences8Bit = true // 8-bit C1 forms (0x80-0x9F), not valid UTF-8
|
||||
```
|
||||
|
||||
For ESC-initiated (7-bit) control strings, only 7-bit terminators are recognized.
|
||||
For C1-initiated (8-bit) control strings, only C1 ST (`0x9C`) is recognized as ST.
|
||||
|
||||
We implement [ECMA-48](https://ecma-international.org/publications-and-standards/standards/ecma-48/) control codes in both 7-bit and 8-bit representations. 8-bit control codes are not UTF-8 encoded and are not valid UTF-8, caveat emptor.
|
||||
|
||||
### Benchmarks
|
||||
|
||||
```
|
||||
goos: darwin
|
||||
goarch: arm64
|
||||
pkg: github.com/clipperhouse/uax29/graphemes/comparative
|
||||
cpu: Apple M2
|
||||
|
||||
BenchmarkGraphemesMixed/clipperhouse/uax29-8 142635 ns/op 245.12 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkGraphemesMixed/rivo/uniseg-8 2018284 ns/op 17.32 MB/s 0 B/op 0 allocs/op
|
||||
|
||||
BenchmarkGraphemesASCII/clipperhouse/uax29-8 8846 ns/op 508.73 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkGraphemesASCII/rivo/uniseg-8 366760 ns/op 12.27 MB/s 0 B/op 0 allocs/op
|
||||
```
|
||||
|
||||
### Invalid inputs
|
||||
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package graphemes
|
||||
|
||||
// ansiEscapeLength returns the byte length of a valid 7-bit ANSI escape
|
||||
// sequence at the start of data, or 0 if none.
|
||||
//
|
||||
// Recognized forms (ECMA-48 / ISO 6429):
|
||||
// - CSI: ESC [ then parameter bytes (0x30-0x3F), intermediate (0x20-0x2F), final (0x40-0x7E)
|
||||
// - OSC: ESC ] then payload until BEL (0x07), 7-bit ST (ESC \), CAN (0x18), or SUB (0x1A)
|
||||
// - DCS, SOS, PM, APC: ESC P/X/^/_ then payload until 7-bit ST (ESC \), CAN, or SUB
|
||||
// - Two-byte: ESC + Fe/Fs (0x40-0x7E excluding above), or Fp (0x30-0x3F), or nF (0x20-0x2F then final)
|
||||
func ansiEscapeLength[T ~string | ~[]byte](data T) int {
|
||||
n := len(data)
|
||||
if n < 2 || data[0] != esc {
|
||||
return 0
|
||||
}
|
||||
|
||||
b1 := data[1]
|
||||
switch b1 {
|
||||
case '[': // CSI
|
||||
body := csiBodyLength(data[2:])
|
||||
if body == 0 {
|
||||
return 0
|
||||
}
|
||||
return 2 + body
|
||||
case ']': // OSC - allows BEL or 7-bit ST terminator
|
||||
body := oscLength(data[2:])
|
||||
if body < 0 {
|
||||
return 0
|
||||
}
|
||||
return 2 + body
|
||||
case 'P', 'X', '^', '_': // DCS, SOS, PM, APC
|
||||
body := stSequenceLength(data[2:])
|
||||
if body < 0 {
|
||||
return 0
|
||||
}
|
||||
return 2 + body
|
||||
}
|
||||
|
||||
if b1 >= 0x40 && b1 <= 0x7E {
|
||||
// Fe/Fs two-byte; [ ] P X ^ _ handled above
|
||||
return 2
|
||||
}
|
||||
if b1 >= 0x30 && b1 <= 0x3F {
|
||||
// Fp (private) two-byte
|
||||
return 2
|
||||
}
|
||||
if b1 >= 0x20 && b1 <= 0x2F {
|
||||
// nF: intermediates then one final (0x30-0x7E)
|
||||
i := 2
|
||||
for i < n && data[i] >= 0x20 && data[i] <= 0x2F {
|
||||
i++
|
||||
}
|
||||
if i < n && data[i] >= 0x30 && data[i] <= 0x7E {
|
||||
return i + 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// csiBodyLength returns the length of the CSI body (param/intermediate/final bytes).
|
||||
// data is the slice after "ESC [".
|
||||
// Per ECMA-48, the CSI body has the form:
|
||||
//
|
||||
// parameters (0x30–0x3F)*, intermediates (0x20–0x2F)*, final (0x40–0x7E)
|
||||
//
|
||||
// Once an intermediate byte is seen, subsequent parameter bytes are invalid.
|
||||
func csiBodyLength[T ~string | ~[]byte](data T) int {
|
||||
seenIntermediate := false
|
||||
for i := 0; i < len(data); i++ {
|
||||
b := data[i]
|
||||
if b >= 0x30 && b <= 0x3F {
|
||||
if seenIntermediate {
|
||||
return 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
if b >= 0x20 && b <= 0x2F {
|
||||
seenIntermediate = true
|
||||
continue
|
||||
}
|
||||
if b >= 0x40 && b <= 0x7E {
|
||||
return i + 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// oscLength returns the length of the OSC body.
|
||||
// data is the slice after "ESC ]".
|
||||
//
|
||||
// Returns:
|
||||
// - n >= 0: consumed body length (includes BEL/ST terminator when present)
|
||||
// - -1: not terminated in the provided data
|
||||
//
|
||||
// OSC accepts BEL (0x07) or 7-bit ST (ESC \) as terminators by widespread convention.
|
||||
// Per ECMA-48, CAN (0x18) and SUB (0x1A) cancel the control string; in that
|
||||
// case they are not part of the OSC sequence length.
|
||||
func oscLength[T ~string | ~[]byte](data T) int {
|
||||
for i := 0; i < len(data); i++ {
|
||||
b := data[i]
|
||||
if b == bel {
|
||||
return i + 1
|
||||
}
|
||||
if b == can || b == sub {
|
||||
return i
|
||||
}
|
||||
if b == esc && i+1 < len(data) && data[i+1] == '\\' {
|
||||
return i + 2
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// stSequenceLength returns the length of a control-string body.
|
||||
// data is the slice after "ESC x".
|
||||
//
|
||||
// Returns:
|
||||
// - n >= 0: consumed body length (includes ST terminator when present)
|
||||
// - -1: not terminated in the provided data
|
||||
//
|
||||
// Used for DCS, SOS, PM, and APC, which per ECMA-48 terminate with ST.
|
||||
// ST here is the 7-bit form (ESC \).
|
||||
// CAN (0x18) and SUB (0x1A) cancel the control string; in that case they are
|
||||
// not part of the sequence length.
|
||||
func stSequenceLength[T ~string | ~[]byte](data T) int {
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] == can || data[i] == sub {
|
||||
return i
|
||||
}
|
||||
if data[i] == esc && i+1 < len(data) && data[i+1] == '\\' {
|
||||
return i + 2
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
package graphemes
|
||||
|
||||
// ansiEscapeLength8Bit returns the byte length of a valid 8-bit C1 ANSI
|
||||
// sequence at the start of data, or 0 if none.
|
||||
//
|
||||
// Recognized forms (ECMA-48 / ISO 6429):
|
||||
// - C1 CSI (0x9B) body as parameter/intermediate/final bytes
|
||||
// - C1 OSC (0x9D) body terminated by BEL, C1 ST, CAN, or SUB
|
||||
// - C1 DCS/SOS/PM/APC (0x90/0x98/0x9E/0x9F) body terminated by C1 ST, CAN, or SUB
|
||||
// - Standalone C1 controls (0x80..0x9F not listed above): single byte
|
||||
func ansiEscapeLength8Bit[T ~string | ~[]byte](data T) int {
|
||||
if len(data) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
switch data[0] {
|
||||
case 0x9B: // C1 CSI
|
||||
body := csiBodyLength(data[1:])
|
||||
if body == 0 {
|
||||
return 0
|
||||
}
|
||||
return 1 + body
|
||||
case 0x9D: // C1 OSC
|
||||
body := oscLengthC1(data[1:])
|
||||
if body < 0 {
|
||||
return 0
|
||||
}
|
||||
return 1 + body
|
||||
case 0x90, 0x98, 0x9E, 0x9F: // C1 DCS, SOS, PM, APC
|
||||
body := stSequenceLengthC1(data[1:])
|
||||
if body < 0 {
|
||||
return 0
|
||||
}
|
||||
return 1 + body
|
||||
default:
|
||||
if data[0] >= 0x80 && data[0] <= 0x9F {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// oscLengthC1 returns the length of a C1 OSC body.
|
||||
// data is the slice after the C1 OSC initiator (0x9D).
|
||||
//
|
||||
// Returns:
|
||||
// - n >= 0: consumed body length (includes BEL/ST terminator when present)
|
||||
// - -1: not terminated in the provided data
|
||||
//
|
||||
// Terminators: BEL (0x07) or C1 ST (0x9C).
|
||||
// CAN (0x18) and SUB (0x1A) cancel the control string.
|
||||
func oscLengthC1[T ~string | ~[]byte](data T) int {
|
||||
for i := 0; i < len(data); i++ {
|
||||
b := data[i]
|
||||
if b == bel || b == st {
|
||||
return i + 1
|
||||
}
|
||||
if b == can || b == sub {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// stSequenceLengthC1 parses DCS/SOS/PM/APC bodies that terminate with C1 ST
|
||||
// (0x9C), or are canceled by CAN/SUB.
|
||||
func stSequenceLengthC1[T ~string | ~[]byte](data T) int {
|
||||
for i := 0; i < len(data); i++ {
|
||||
b := data[i]
|
||||
if b == can || b == sub {
|
||||
return i
|
||||
}
|
||||
if b == st {
|
||||
return i + 1
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
+129
-13
@@ -1,9 +1,44 @@
|
||||
package graphemes
|
||||
|
||||
import "github.com/clipperhouse/uax29/v2/internal/iterators"
|
||||
import "unicode/utf8"
|
||||
|
||||
type Iterator[T iterators.Stringish] struct {
|
||||
*iterators.Iterator[T]
|
||||
// FromString returns an iterator for the grapheme clusters in the input string.
|
||||
// Iterate while Next() is true, and access the grapheme via Value().
|
||||
func FromString(s string) *Iterator[string] {
|
||||
return &Iterator[string]{
|
||||
split: splitFuncString,
|
||||
data: s,
|
||||
}
|
||||
}
|
||||
|
||||
// FromBytes returns an iterator for the grapheme clusters in the input bytes.
|
||||
// Iterate while Next() is true, and access the grapheme via Value().
|
||||
func FromBytes(b []byte) *Iterator[[]byte] {
|
||||
return &Iterator[[]byte]{
|
||||
split: splitFuncBytes,
|
||||
data: b,
|
||||
}
|
||||
}
|
||||
|
||||
// Iterator is a generic iterator for grapheme clusters in strings or byte slices,
|
||||
// with an ASCII hot path optimization.
|
||||
type Iterator[T ~string | ~[]byte] struct {
|
||||
split func(T, bool) (int, T, error)
|
||||
data T
|
||||
pos int
|
||||
start int
|
||||
// AnsiEscapeSequences treats 7-bit ANSI escape sequences (ECMA-48) as
|
||||
// single grapheme clusters when true. The default is false.
|
||||
//
|
||||
// 8-bit controls are not enabled by this option. See [AnsiEscapeSequences8Bit].
|
||||
AnsiEscapeSequences bool
|
||||
// AnsiEscapeSequences8Bit treats 8-bit C1 ANSI escape sequences (ECMA-48) as single
|
||||
// grapheme clusters when true. The default is false.
|
||||
//
|
||||
// 8-bit control bytes are not UTF-8 encoded, i.e. not valid UTF-8. If you
|
||||
// choose this option, you are choosing to interpret non-UTF-8 data, caveat
|
||||
// emptor.
|
||||
AnsiEscapeSequences8Bit bool
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -11,18 +46,99 @@ var (
|
||||
splitFuncBytes = splitFunc[[]byte]
|
||||
)
|
||||
|
||||
// FromString returns an iterator for the grapheme clusters in the input string.
|
||||
// Iterate while Next() is true, and access the grapheme via Value().
|
||||
func FromString(s string) Iterator[string] {
|
||||
return Iterator[string]{
|
||||
iterators.New(splitFuncString, s),
|
||||
const (
|
||||
esc = 0x1B
|
||||
cr = 0x0D
|
||||
bel = 0x07
|
||||
can = 0x18
|
||||
sub = 0x1A
|
||||
st = 0x9C
|
||||
)
|
||||
|
||||
// Next advances the iterator to the next grapheme cluster.
|
||||
// Returns false when there are no more grapheme clusters.
|
||||
func (iter *Iterator[T]) Next() bool {
|
||||
if iter.pos >= len(iter.data) {
|
||||
return false
|
||||
}
|
||||
iter.start = iter.pos
|
||||
|
||||
b := iter.data[iter.pos]
|
||||
if iter.AnsiEscapeSequences && b == esc {
|
||||
if a := ansiEscapeLength(iter.data[iter.pos:]); a > 0 {
|
||||
iter.pos += a
|
||||
return true
|
||||
}
|
||||
}
|
||||
if iter.AnsiEscapeSequences8Bit && b >= 0x80 && b <= 0x9F {
|
||||
if a := ansiEscapeLength8Bit(iter.data[iter.pos:]); a > 0 {
|
||||
iter.pos += a
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// ASCII hot path: any ASCII is one grapheme when next byte is ASCII or end.
|
||||
if b < utf8.RuneSelf && b != cr {
|
||||
if iter.pos+1 >= len(iter.data) || iter.data[iter.pos+1] < utf8.RuneSelf {
|
||||
iter.pos++
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to UAX29 grapheme parsing
|
||||
remaining := iter.data[iter.pos:]
|
||||
advance, _, err := iter.split(remaining, true)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if advance <= 0 {
|
||||
panic("splitFunc returned a zero or negative advance")
|
||||
}
|
||||
iter.pos += advance
|
||||
if iter.pos > len(iter.data) {
|
||||
panic("splitFunc advanced beyond end of data")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// FromBytes returns an iterator for the grapheme clusters in the input bytes.
|
||||
// Iterate while Next() is true, and access the grapheme via Value().
|
||||
func FromBytes(b []byte) Iterator[[]byte] {
|
||||
return Iterator[[]byte]{
|
||||
iterators.New(splitFuncBytes, b),
|
||||
// Value returns the current grapheme cluster.
|
||||
func (iter *Iterator[T]) Value() T {
|
||||
return iter.data[iter.start:iter.pos]
|
||||
}
|
||||
|
||||
// Start returns the byte position of the current grapheme in the original data.
|
||||
func (iter *Iterator[T]) Start() int {
|
||||
return iter.start
|
||||
}
|
||||
|
||||
// End returns the byte position after the current grapheme in the original data.
|
||||
func (iter *Iterator[T]) End() int {
|
||||
return iter.pos
|
||||
}
|
||||
|
||||
// Reset resets the iterator to the beginning of the data.
|
||||
func (iter *Iterator[T]) Reset() {
|
||||
iter.start = 0
|
||||
iter.pos = 0
|
||||
}
|
||||
|
||||
// SetText sets the data for the iterator to operate on, and resets all state.
|
||||
func (iter *Iterator[T]) SetText(data T) {
|
||||
iter.data = data
|
||||
iter.start = 0
|
||||
iter.pos = 0
|
||||
}
|
||||
|
||||
// First returns the first grapheme cluster without advancing the iterator.
|
||||
func (iter *Iterator[T]) First() T {
|
||||
if len(iter.data) == 0 {
|
||||
return iter.data
|
||||
}
|
||||
|
||||
// Use a copy to leverage Next()'s ASCII optimization
|
||||
cp := *iter
|
||||
cp.pos = 0
|
||||
cp.start = 0
|
||||
cp.Next()
|
||||
return cp.Value()
|
||||
}
|
||||
|
||||
+39
-8
@@ -2,8 +2,6 @@ package graphemes
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
|
||||
"github.com/clipperhouse/uax29/v2/internal/iterators"
|
||||
)
|
||||
|
||||
// is determines if lookup intersects propert(ies)
|
||||
@@ -13,12 +11,22 @@ func (lookup property) is(properties property) bool {
|
||||
|
||||
const _Ignore = _Extend
|
||||
|
||||
// incbState tracks state for GB9c rule (Indic conjunct clusters)
|
||||
// Pattern: Consonant (Extend|Linker)* Linker (Extend|Linker)* × Consonant
|
||||
type incbState int
|
||||
|
||||
const (
|
||||
incbNone incbState = iota // initial/reset
|
||||
incbConsonant // seen Consonant, awaiting Linker
|
||||
incbLinker // seen Consonant and Linker (conjunct ready)
|
||||
)
|
||||
|
||||
// SplitFunc is a bufio.SplitFunc implementation of Unicode grapheme cluster segmentation, for use with bufio.Scanner.
|
||||
//
|
||||
// See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
|
||||
var SplitFunc bufio.SplitFunc = splitFunc[[]byte]
|
||||
|
||||
func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T, err error) {
|
||||
func splitFunc[T ~string | ~[]byte](data T, atEOF bool) (advance int, token T, err error) {
|
||||
var empty T
|
||||
if len(data) == 0 {
|
||||
return 0, empty, nil
|
||||
@@ -30,6 +38,9 @@ func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T,
|
||||
var lastLastExIgnore property = 0 // "last one before that"
|
||||
var regionalIndicatorCount int
|
||||
|
||||
// GB9c state: tracking Indic conjunct clusters
|
||||
var incb incbState
|
||||
|
||||
// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
|
||||
// to the right of the ×, from which we look back or forward
|
||||
|
||||
@@ -76,6 +87,23 @@ func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T,
|
||||
lastExIgnore = last
|
||||
}
|
||||
|
||||
// Update GB9c state based on what we just advanced past
|
||||
if last.is(_InCBConsonant | _InCBLinker | _InCBExtend) {
|
||||
switch {
|
||||
case last.is(_InCBConsonant):
|
||||
if incb != incbLinker {
|
||||
incb = incbConsonant
|
||||
}
|
||||
case last.is(_InCBLinker):
|
||||
if incb >= incbConsonant {
|
||||
incb = incbLinker
|
||||
}
|
||||
// case last.is(_InCBExtend): stay in current state
|
||||
}
|
||||
} else {
|
||||
incb = incbNone
|
||||
}
|
||||
|
||||
current, w = lookup(data[pos:])
|
||||
if w == 0 {
|
||||
if atEOF {
|
||||
@@ -141,11 +169,14 @@ func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T,
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB9c
|
||||
// TODO(clipperhouse):
|
||||
// It appears to be added in Unicode 15.1.0:
|
||||
// https://unicode.org/versions/Unicode15.1.0/#Migration
|
||||
// This package currently supports Unicode 15.0.0, so
|
||||
// out of scope for now
|
||||
// Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker.
|
||||
if incb == incbLinker && current.is(_InCBConsonant) {
|
||||
// After matching the pattern, reset state to start tracking a new pattern
|
||||
// The current Consonant becomes the start of the new pattern
|
||||
incb = incbConsonant
|
||||
pos += w
|
||||
continue
|
||||
}
|
||||
|
||||
// https://unicode.org/reports/tr29/#GB11
|
||||
if current.is(_ExtendedPictographic) && last.is(_ZWJ) && lastLastExIgnore.is(_ExtendedPictographic) {
|
||||
|
||||
+1323
-1015
File diff suppressed because it is too large
Load Diff
-85
@@ -1,85 +0,0 @@
|
||||
package iterators
|
||||
|
||||
type Stringish interface {
|
||||
[]byte | string
|
||||
}
|
||||
|
||||
type SplitFunc[T Stringish] func(T, bool) (int, T, error)
|
||||
|
||||
// Iterator is a generic iterator for words that are either []byte or string.
|
||||
// Iterate while Next() is true, and access the word via Value().
|
||||
type Iterator[T Stringish] struct {
|
||||
split SplitFunc[T]
|
||||
data T
|
||||
start int
|
||||
pos int
|
||||
}
|
||||
|
||||
// New creates a new Iterator for the given data and SplitFunc.
|
||||
func New[T Stringish](split SplitFunc[T], data T) *Iterator[T] {
|
||||
return &Iterator[T]{
|
||||
split: split,
|
||||
data: data,
|
||||
}
|
||||
}
|
||||
|
||||
// SetText sets the text for the iterator to operate on, and resets all state.
|
||||
func (iter *Iterator[T]) SetText(data T) {
|
||||
iter.data = data
|
||||
iter.start = 0
|
||||
iter.pos = 0
|
||||
}
|
||||
|
||||
// Split sets the SplitFunc for the Iterator.
|
||||
func (iter *Iterator[T]) Split(split SplitFunc[T]) {
|
||||
iter.split = split
|
||||
}
|
||||
|
||||
// Next advances the iterator to the next token. It returns false when there
|
||||
// are no remaining tokens or an error occurred.
|
||||
func (iter *Iterator[T]) Next() bool {
|
||||
if iter.pos == len(iter.data) {
|
||||
return false
|
||||
}
|
||||
if iter.pos > len(iter.data) {
|
||||
panic("SplitFunc advanced beyond the end of the data")
|
||||
}
|
||||
|
||||
iter.start = iter.pos
|
||||
|
||||
advance, _, err := iter.split(iter.data[iter.pos:], true)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if advance <= 0 {
|
||||
panic("SplitFunc returned a zero or negative advance")
|
||||
}
|
||||
|
||||
iter.pos += advance
|
||||
if iter.pos > len(iter.data) {
|
||||
panic("SplitFunc advanced beyond the end of the data")
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Value returns the current token.
|
||||
func (iter *Iterator[T]) Value() T {
|
||||
return iter.data[iter.start:iter.pos]
|
||||
}
|
||||
|
||||
// Start returns the byte position of the current token in the original data.
|
||||
func (iter *Iterator[T]) Start() int {
|
||||
return iter.start
|
||||
}
|
||||
|
||||
// End returns the byte position after the current token in the original data.
|
||||
func (iter *Iterator[T]) End() int {
|
||||
return iter.pos
|
||||
}
|
||||
|
||||
// Reset resets the iterator to the beginning of the data.
|
||||
func (iter *Iterator[T]) Reset() {
|
||||
iter.start = 0
|
||||
iter.pos = 0
|
||||
}
|
||||
Vendored
+1
-2
@@ -21,10 +21,9 @@ github.com/cenkalti/backoff/v5
|
||||
# github.com/cespare/xxhash/v2 v2.3.0
|
||||
## explicit; go 1.11
|
||||
github.com/cespare/xxhash/v2
|
||||
# github.com/clipperhouse/uax29/v2 v2.2.0
|
||||
# github.com/clipperhouse/uax29/v2 v2.7.0
|
||||
## explicit; go 1.18
|
||||
github.com/clipperhouse/uax29/v2/graphemes
|
||||
github.com/clipperhouse/uax29/v2/internal/iterators
|
||||
# github.com/containerd/errdefs v1.0.0
|
||||
## explicit; go 1.20
|
||||
github.com/containerd/errdefs
|
||||
|
||||
Reference in New Issue
Block a user