vendor: github.com/clipperhouse/uax29/v2 v2.7.0

- Adds Unicode 17 support - Fast paths for ASCII - ANSI escape sequences as grapheme clusters - Implement UTF-8 C1 control sequences - Implement 8-bit ANSI Breaking change The returned iterator type from FromString() and FromBytes() is now a pointer. This will not present a problem if you are just using it in the typical way, which is assigning to a local variable and iterating. If you are embedding this iterator into another object, and therefore declaring its type, this change might break your compilation. You’ll need to change to a pointer type. Apologies if so — this seems like a small enough change that a v3 would be a bit much. full diff: https://github.com/clipperhouse/uax29/compare/v2.2.0...v2.7.0 Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
2026-09-26 17:30:51 -04:00 · 2026-09-07 09:21:26 +02:00
parent d306b9d600
commit 676ca0067b
10 changed files with 1775 additions and 1151 deletions
@@ -75,7 +75,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
+	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/docker/go-events v0.1.0 // indirect
 	github.com/docker/go-metrics v0.1.0 // indirect
@@ -12,8 +12,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
-github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
+github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
+github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
 github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
 github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
@@ -1,32 +1,34 @@
-An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode version 15.0.0.
+An implementation of grapheme cluster boundaries from [Unicode text segmentation](https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries) (UAX 29), for Unicode 17.
+
+[![Documentation](https://pkg.go.dev/badge/github.com/clipperhouse/uax29/v2/graphemes.svg)](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
+![Tests](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
+![Fuzz](https://github.com/clipperhouse/uax29/actions/workflows/gofuzz.yml/badge.svg)

 ## Quick start

 ```
-go get "github.com/clipperhouse/uax29/v2/graphemes"
+go get github.com/clipperhouse/uax29/v2/graphemes
 ```

 ```go
 import "github.com/clipperhouse/uax29/v2/graphemes"

 text := "Hello, 世界. Nice dog! 👍🐶"
+g := graphemes.FromString(text)

-tokens := graphemes.FromString(text)
-
-for tokens.Next() {                     // Next() returns true until end of data
-	fmt.Println(tokens.Value())         // Do something with the current grapheme
+for g.Next() {                     // Next() returns true until end of data
+	fmt.Println(g.Value())         // Do something with the current grapheme
 }
 ```

-[![Documentation](https://pkg.go.dev/badge/github.com/clipperhouse/uax29/v2/graphemes.svg)](https://pkg.go.dev/github.com/clipperhouse/uax29/v2/graphemes)
-
 _A grapheme is a “single visible character”, which might be a simple as a single letter, or a complex emoji that consists of several Unicode code points._

 ## Conformance

-We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Tests29). Status:
+We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-36.html#Tests29).

-![Go](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
+![Tests](https://github.com/clipperhouse/uax29/actions/workflows/gotest.yml/badge.svg)
+![Fuzz](https://github.com/clipperhouse/uax29/actions/workflows/gofuzz.yml/badge.svg)

 ## APIs

@@ -34,11 +36,10 @@ We use the Unicode [test suite](https://unicode.org/reports/tr41/tr41-26.html#Te

 ```go
 text := "Hello, 世界. Nice dog! 👍🐶"
+g := graphemes.FromString(text)

-tokens := graphemes.FromString(text)
-
-for tokens.Next() {                     // Next() returns true until end of data
-	fmt.Println(tokens.Value())         // Do something with the current grapheme
+for g.Next() {                     // Next() returns true until end of data
+	fmt.Println(g.Value())         // Do something with the current grapheme
 }
 ```

@@ -47,15 +48,15 @@ for tokens.Next() {                     // Next() returns true until end of data
 `FromReader` embeds a [`bufio.Scanner`](https://pkg.go.dev/bufio#Scanner), so just use those methods.

 ```go
-r := getYourReader()                        // from a file or network maybe
-tokens := graphemes.FromReader(r)
+r := getYourReader()                    // from a file or network maybe
+g := graphemes.FromReader(r)

-for tokens.Scan() {                         // Scan() returns true until error or EOF
-	fmt.Println(tokens.Text())              // Do something with the current grapheme
+for g.Scan() {                         // Scan() returns true until error or EOF
+	fmt.Println(g.Text())              // Do something with the current grapheme
 }

-if tokens.Err() != nil {                    // Check the error
-	log.Fatal(tokens.Err())
+if g.Err() != nil {                    // Check the error
+	log.Fatal(g.Err())
 }
 ```

@@ -64,16 +65,53 @@ if tokens.Err() != nil {                    // Check the error
 ```go
 b := []byte("Hello, 世界. Nice dog! 👍🐶")

-tokens := graphemes.FromBytes(b)
+g := graphemes.FromBytes(b)

-for tokens.Next() {                     // Next() returns true until end of data
-	fmt.Println(tokens.Value())         // Do something with the current grapheme
+for g.Next() {                     // Next() returns true until end of data
+	fmt.Println(g.Value())         // Do something with the current grapheme
 }
 ```

-### Performance
+### ANSI escape sequences

-On a Mac M2 laptop, we see around 200MB/s, or around 100 million graphemes per second. You should see ~constant memory, and no allocations.
+By the UAX 29 specification, ANSI escape sequences are not grapheme clusters. To treat 7-bit ANSI escape sequences as a single cluster, set `AnsiEscapeSequences` to true.
+
+```go
+text := "Hello, \x1b[31mworld\x1b[0m!"
+g := graphemes.FromString(text)
+g.AnsiEscapeSequences = true
+
+for g.Next() {
+	fmt.Println(g.Value())
+}
+```
+
+To also parse 8-bit C1 controls (non-UTF-8 bytes), set `AnsiEscapeSequences8Bit` to true.
+
+```go
+g.AnsiEscapeSequences = true     // 7-bit forms (ESC ...)
+g.AnsiEscapeSequences8Bit = true // 8-bit C1 forms (0x80-0x9F), not valid UTF-8
+```
+
+For ESC-initiated (7-bit) control strings, only 7-bit terminators are recognized.
+For C1-initiated (8-bit) control strings, only C1 ST (`0x9C`) is recognized as ST.
+
+We implement [ECMA-48](https://ecma-international.org/publications-and-standards/standards/ecma-48/) control codes in both 7-bit and 8-bit representations. 8-bit control codes are not UTF-8 encoded and are not valid UTF-8, caveat emptor.
+
+### Benchmarks
+
+```
+goos: darwin
+goarch: arm64
+pkg: github.com/clipperhouse/uax29/graphemes/comparative
+cpu: Apple M2
+
+BenchmarkGraphemesMixed/clipperhouse/uax29-8  	    142635 ns/op	 245.12 MB/s    0 B/op	   0 allocs/op
+BenchmarkGraphemesMixed/rivo/uniseg-8         	   2018284 ns/op	  17.32 MB/s    0 B/op	   0 allocs/op
+
+BenchmarkGraphemesASCII/clipperhouse/uax29-8  	      8846 ns/op	 508.73 MB/s    0 B/op	   0 allocs/op
+BenchmarkGraphemesASCII/rivo/uniseg-8         	    366760 ns/op	  12.27 MB/s    0 B/op	   0 allocs/op
+```

 ### Invalid inputs

@@ -0,0 +1,138 @@
+package graphemes
+
+// ansiEscapeLength returns the byte length of a valid 7-bit ANSI escape
+// sequence at the start of data, or 0 if none.
+//
+// Recognized forms (ECMA-48 / ISO 6429):
+//   - CSI: ESC [ then parameter bytes (0x30-0x3F), intermediate (0x20-0x2F), final (0x40-0x7E)
+//   - OSC: ESC ] then payload until BEL (0x07), 7-bit ST (ESC \), CAN (0x18), or SUB (0x1A)
+//   - DCS, SOS, PM, APC: ESC P/X/^/_ then payload until 7-bit ST (ESC \), CAN, or SUB
+//   - Two-byte: ESC + Fe/Fs (0x40-0x7E excluding above), or Fp (0x30-0x3F), or nF (0x20-0x2F then final)
+func ansiEscapeLength[T ~string | ~[]byte](data T) int {
+	n := len(data)
+	if n < 2 || data[0] != esc {
+		return 0
+	}
+
+	b1 := data[1]
+	switch b1 {
+	case '[': // CSI
+		body := csiBodyLength(data[2:])
+		if body == 0 {
+			return 0
+		}
+		return 2 + body
+	case ']': // OSC - allows BEL or 7-bit ST terminator
+		body := oscLength(data[2:])
+		if body < 0 {
+			return 0
+		}
+		return 2 + body
+	case 'P', 'X', '^', '_': // DCS, SOS, PM, APC
+		body := stSequenceLength(data[2:])
+		if body < 0 {
+			return 0
+		}
+		return 2 + body
+	}
+
+	if b1 >= 0x40 && b1 <= 0x7E {
+		// Fe/Fs two-byte; [ ] P X ^ _ handled above
+		return 2
+	}
+	if b1 >= 0x30 && b1 <= 0x3F {
+		// Fp (private) two-byte
+		return 2
+	}
+	if b1 >= 0x20 && b1 <= 0x2F {
+		// nF: intermediates then one final (0x30-0x7E)
+		i := 2
+		for i < n && data[i] >= 0x20 && data[i] <= 0x2F {
+			i++
+		}
+		if i < n && data[i] >= 0x30 && data[i] <= 0x7E {
+			return i + 1
+		}
+		return 0
+	}
+
+	return 0
+}
+
+// csiBodyLength returns the length of the CSI body (param/intermediate/final bytes).
+// data is the slice after "ESC [".
+// Per ECMA-48, the CSI body has the form:
+//
+//	parameters (0x30–0x3F)*, intermediates (0x20–0x2F)*, final (0x40–0x7E)
+//
+// Once an intermediate byte is seen, subsequent parameter bytes are invalid.
+func csiBodyLength[T ~string | ~[]byte](data T) int {
+	seenIntermediate := false
+	for i := 0; i < len(data); i++ {
+		b := data[i]
+		if b >= 0x30 && b <= 0x3F {
+			if seenIntermediate {
+				return 0
+			}
+			continue
+		}
+		if b >= 0x20 && b <= 0x2F {
+			seenIntermediate = true
+			continue
+		}
+		if b >= 0x40 && b <= 0x7E {
+			return i + 1
+		}
+		return 0
+	}
+	return 0
+}
+
+// oscLength returns the length of the OSC body.
+// data is the slice after "ESC ]".
+//
+// Returns:
+//   - n >= 0: consumed body length (includes BEL/ST terminator when present)
+//   - -1: not terminated in the provided data
+//
+// OSC accepts BEL (0x07) or 7-bit ST (ESC \) as terminators by widespread convention.
+// Per ECMA-48, CAN (0x18) and SUB (0x1A) cancel the control string; in that
+// case they are not part of the OSC sequence length.
+func oscLength[T ~string | ~[]byte](data T) int {
+	for i := 0; i < len(data); i++ {
+		b := data[i]
+		if b == bel {
+			return i + 1
+		}
+		if b == can || b == sub {
+			return i
+		}
+		if b == esc && i+1 < len(data) && data[i+1] == '\\' {
+			return i + 2
+		}
+	}
+	return -1
+}
+
+// stSequenceLength returns the length of a control-string body.
+// data is the slice after "ESC x".
+//
+// Returns:
+//   - n >= 0: consumed body length (includes ST terminator when present)
+//   - -1: not terminated in the provided data
+//
+// Used for DCS, SOS, PM, and APC, which per ECMA-48 terminate with ST.
+// ST here is the 7-bit form (ESC \).
+// CAN (0x18) and SUB (0x1A) cancel the control string; in that case they are
+// not part of the sequence length.
+func stSequenceLength[T ~string | ~[]byte](data T) int {
+	for i := 0; i < len(data); i++ {
+		if data[i] == can || data[i] == sub {
+			return i
+		}
+		if data[i] == esc && i+1 < len(data) && data[i+1] == '\\' {
+			return i + 2
+		}
+	}
+	return -1
+}
@@ -0,0 +1,79 @@
+package graphemes
+
+// ansiEscapeLength8Bit returns the byte length of a valid 8-bit C1 ANSI
+// sequence at the start of data, or 0 if none.
+//
+// Recognized forms (ECMA-48 / ISO 6429):
+//   - C1 CSI (0x9B) body as parameter/intermediate/final bytes
+//   - C1 OSC (0x9D) body terminated by BEL, C1 ST, CAN, or SUB
+//   - C1 DCS/SOS/PM/APC (0x90/0x98/0x9E/0x9F) body terminated by C1 ST, CAN, or SUB
+//   - Standalone C1 controls (0x80..0x9F not listed above): single byte
+func ansiEscapeLength8Bit[T ~string | ~[]byte](data T) int {
+	if len(data) == 0 {
+		return 0
+	}
+
+	switch data[0] {
+	case 0x9B: // C1 CSI
+		body := csiBodyLength(data[1:])
+		if body == 0 {
+			return 0
+		}
+		return 1 + body
+	case 0x9D: // C1 OSC
+		body := oscLengthC1(data[1:])
+		if body < 0 {
+			return 0
+		}
+		return 1 + body
+	case 0x90, 0x98, 0x9E, 0x9F: // C1 DCS, SOS, PM, APC
+		body := stSequenceLengthC1(data[1:])
+		if body < 0 {
+			return 0
+		}
+		return 1 + body
+	default:
+		if data[0] >= 0x80 && data[0] <= 0x9F {
+			return 1
+		}
+	}
+
+	return 0
+}
+
+// oscLengthC1 returns the length of a C1 OSC body.
+// data is the slice after the C1 OSC initiator (0x9D).
+//
+// Returns:
+//   - n >= 0: consumed body length (includes BEL/ST terminator when present)
+//   - -1: not terminated in the provided data
+//
+// Terminators: BEL (0x07) or C1 ST (0x9C).
+// CAN (0x18) and SUB (0x1A) cancel the control string.
+func oscLengthC1[T ~string | ~[]byte](data T) int {
+	for i := 0; i < len(data); i++ {
+		b := data[i]
+		if b == bel || b == st {
+			return i + 1
+		}
+		if b == can || b == sub {
+			return i
+		}
+	}
+	return -1
+}
+
+// stSequenceLengthC1 parses DCS/SOS/PM/APC bodies that terminate with C1 ST
+// (0x9C), or are canceled by CAN/SUB.
+func stSequenceLengthC1[T ~string | ~[]byte](data T) int {
+	for i := 0; i < len(data); i++ {
+		b := data[i]
+		if b == can || b == sub {
+			return i
+		}
+		if b == st {
+			return i + 1
+		}
+	}
+	return -1
+}
@@ -1,9 +1,44 @@
 package graphemes

-import "github.com/clipperhouse/uax29/v2/internal/iterators"
+import "unicode/utf8"

-type Iterator[T iterators.Stringish] struct {
-	*iterators.Iterator[T]
+// FromString returns an iterator for the grapheme clusters in the input string.
+// Iterate while Next() is true, and access the grapheme via Value().
+func FromString(s string) *Iterator[string] {
+	return &Iterator[string]{
+		split: splitFuncString,
+		data:  s,
+	}
+}
+
+// FromBytes returns an iterator for the grapheme clusters in the input bytes.
+// Iterate while Next() is true, and access the grapheme via Value().
+func FromBytes(b []byte) *Iterator[[]byte] {
+	return &Iterator[[]byte]{
+		split: splitFuncBytes,
+		data:  b,
+	}
+}
+
+// Iterator is a generic iterator for grapheme clusters in strings or byte slices,
+// with an ASCII hot path optimization.
+type Iterator[T ~string | ~[]byte] struct {
+	split func(T, bool) (int, T, error)
+	data  T
+	pos   int
+	start int
+	// AnsiEscapeSequences treats 7-bit ANSI escape sequences (ECMA-48) as
+	// single grapheme clusters when true. The default is false.
+	//
+	// 8-bit controls are not enabled by this option. See [AnsiEscapeSequences8Bit].
+	AnsiEscapeSequences bool
+	// AnsiEscapeSequences8Bit treats 8-bit C1 ANSI escape sequences (ECMA-48) as single
+	// grapheme clusters when true. The default is false.
+	//
+	// 8-bit control bytes are not UTF-8 encoded, i.e. not valid UTF-8. If you
+	// choose this option, you are choosing to interpret non-UTF-8 data, caveat
+	// emptor.
+	AnsiEscapeSequences8Bit bool
 }

 var (
@@ -11,18 +46,99 @@ var (
 	splitFuncBytes  = splitFunc[[]byte]
 )

-// FromString returns an iterator for the grapheme clusters in the input string.
-// Iterate while Next() is true, and access the grapheme via Value().
-func FromString(s string) Iterator[string] {
-	return Iterator[string]{
-		iterators.New(splitFuncString, s),
+const (
+	esc = 0x1B
+	cr  = 0x0D
+	bel = 0x07
+	can = 0x18
+	sub = 0x1A
+	st  = 0x9C
+)
+
+// Next advances the iterator to the next grapheme cluster.
+// Returns false when there are no more grapheme clusters.
+func (iter *Iterator[T]) Next() bool {
+	if iter.pos >= len(iter.data) {
+		return false
 	}
+	iter.start = iter.pos
+
+	b := iter.data[iter.pos]
+	if iter.AnsiEscapeSequences && b == esc {
+		if a := ansiEscapeLength(iter.data[iter.pos:]); a > 0 {
+			iter.pos += a
+			return true
+		}
+	}
+	if iter.AnsiEscapeSequences8Bit && b >= 0x80 && b <= 0x9F {
+		if a := ansiEscapeLength8Bit(iter.data[iter.pos:]); a > 0 {
+			iter.pos += a
+			return true
+		}
+	}
+
+	// ASCII hot path: any ASCII is one grapheme when next byte is ASCII or end.
+	if b < utf8.RuneSelf && b != cr {
+		if iter.pos+1 >= len(iter.data) || iter.data[iter.pos+1] < utf8.RuneSelf {
+			iter.pos++
+			return true
+		}
+	}
+
+	// Fall back to UAX29 grapheme parsing
+	remaining := iter.data[iter.pos:]
+	advance, _, err := iter.split(remaining, true)
+	if err != nil {
+		panic(err)
+	}
+	if advance <= 0 {
+		panic("splitFunc returned a zero or negative advance")
+	}
+	iter.pos += advance
+	if iter.pos > len(iter.data) {
+		panic("splitFunc advanced beyond end of data")
+	}
+	return true
 }

-// FromBytes returns an iterator for the grapheme clusters in the input bytes.
-// Iterate while Next() is true, and access the grapheme via Value().
-func FromBytes(b []byte) Iterator[[]byte] {
-	return Iterator[[]byte]{
-		iterators.New(splitFuncBytes, b),
+// Value returns the current grapheme cluster.
+func (iter *Iterator[T]) Value() T {
+	return iter.data[iter.start:iter.pos]
+}
+
+// Start returns the byte position of the current grapheme in the original data.
+func (iter *Iterator[T]) Start() int {
+	return iter.start
+}
+
+// End returns the byte position after the current grapheme in the original data.
+func (iter *Iterator[T]) End() int {
+	return iter.pos
+}
+
+// Reset resets the iterator to the beginning of the data.
+func (iter *Iterator[T]) Reset() {
+	iter.start = 0
+	iter.pos = 0
+}
+
+// SetText sets the data for the iterator to operate on, and resets all state.
+func (iter *Iterator[T]) SetText(data T) {
+	iter.data = data
+	iter.start = 0
+	iter.pos = 0
+}
+
+// First returns the first grapheme cluster without advancing the iterator.
+func (iter *Iterator[T]) First() T {
+	if len(iter.data) == 0 {
+		return iter.data
 	}
+
+	// Use a copy to leverage Next()'s ASCII optimization
+	cp := *iter
+	cp.pos = 0
+	cp.start = 0
+	cp.Next()
+	return cp.Value()
 }
@@ -2,8 +2,6 @@ package graphemes

 import (
 	"bufio"
-
-	"github.com/clipperhouse/uax29/v2/internal/iterators"
 )

 // is determines if lookup intersects propert(ies)
@@ -13,12 +11,22 @@ func (lookup property) is(properties property) bool {

 const _Ignore = _Extend

+// incbState tracks state for GB9c rule (Indic conjunct clusters)
+// Pattern: Consonant (Extend|Linker)* Linker (Extend|Linker)* × Consonant
+type incbState int
+
+const (
+	incbNone      incbState = iota // initial/reset
+	incbConsonant                  // seen Consonant, awaiting Linker
+	incbLinker                     // seen Consonant and Linker (conjunct ready)
+)
+
 // SplitFunc is a bufio.SplitFunc implementation of Unicode grapheme cluster segmentation, for use with bufio.Scanner.
 //
 // See https://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries.
 var SplitFunc bufio.SplitFunc = splitFunc[[]byte]

-func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T, err error) {
+func splitFunc[T ~string | ~[]byte](data T, atEOF bool) (advance int, token T, err error) {
 	var empty T
 	if len(data) == 0 {
 		return 0, empty, nil
@@ -30,6 +38,9 @@ func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T,
 	var lastLastExIgnore property = 0 // "last one before that"
 	var regionalIndicatorCount int

+	// GB9c state: tracking Indic conjunct clusters
+	var incb incbState
+
 	// Rules are usually of the form Cat1 × Cat2; "current" refers to the first property
 	// to the right of the ×, from which we look back or forward

@@ -76,6 +87,23 @@ func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T,
 			lastExIgnore = last
 		}

+		// Update GB9c state based on what we just advanced past
+		if last.is(_InCBConsonant | _InCBLinker | _InCBExtend) {
+			switch {
+			case last.is(_InCBConsonant):
+				if incb != incbLinker {
+					incb = incbConsonant
+				}
+			case last.is(_InCBLinker):
+				if incb >= incbConsonant {
+					incb = incbLinker
+				}
+				// case last.is(_InCBExtend): stay in current state
+			}
+		} else {
+			incb = incbNone
+		}
+
 		current, w = lookup(data[pos:])
 		if w == 0 {
 			if atEOF {
@@ -141,11 +169,14 @@ func splitFunc[T iterators.Stringish](data T, atEOF bool) (advance int, token T,
 		}

 		// https://unicode.org/reports/tr29/#GB9c
-		// TODO(clipperhouse):
-		// It appears to be added in Unicode 15.1.0:
-		// https://unicode.org/versions/Unicode15.1.0/#Migration
-		// This package currently supports Unicode 15.0.0, so
-		// out of scope for now
+		// Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker.
+		if incb == incbLinker && current.is(_InCBConsonant) {
+			// After matching the pattern, reset state to start tracking a new pattern
+			// The current Consonant becomes the start of the new pattern
+			incb = incbConsonant
+			pos += w
+			continue
+		}

 		// https://unicode.org/reports/tr29/#GB11
 		if current.is(_ExtendedPictographic) && last.is(_ZWJ) && lastLastExIgnore.is(_ExtendedPictographic) {
@@ -1,85 +0,0 @@
-package iterators
-
-type Stringish interface {
-	[]byte | string
-}
-
-type SplitFunc[T Stringish] func(T, bool) (int, T, error)
-
-// Iterator is a generic iterator for words that are either []byte or string.
-// Iterate while Next() is true, and access the word via Value().
-type Iterator[T Stringish] struct {
-	split SplitFunc[T]
-	data  T
-	start int
-	pos   int
-}
-
-// New creates a new Iterator for the given data and SplitFunc.
-func New[T Stringish](split SplitFunc[T], data T) *Iterator[T] {
-	return &Iterator[T]{
-		split: split,
-		data:  data,
-	}
-}
-
-// SetText sets the text for the iterator to operate on, and resets all state.
-func (iter *Iterator[T]) SetText(data T) {
-	iter.data = data
-	iter.start = 0
-	iter.pos = 0
-}
-
-// Split sets the SplitFunc for the Iterator.
-func (iter *Iterator[T]) Split(split SplitFunc[T]) {
-	iter.split = split
-}
-
-// Next advances the iterator to the next token. It returns false when there
-// are no remaining tokens or an error occurred.
-func (iter *Iterator[T]) Next() bool {
-	if iter.pos == len(iter.data) {
-		return false
-	}
-	if iter.pos > len(iter.data) {
-		panic("SplitFunc advanced beyond the end of the data")
-	}
-
-	iter.start = iter.pos
-
-	advance, _, err := iter.split(iter.data[iter.pos:], true)
-	if err != nil {
-		panic(err)
-	}
-	if advance <= 0 {
-		panic("SplitFunc returned a zero or negative advance")
-	}
-
-	iter.pos += advance
-	if iter.pos > len(iter.data) {
-		panic("SplitFunc advanced beyond the end of the data")
-	}
-
-	return true
-}
-
-// Value returns the current token.
-func (iter *Iterator[T]) Value() T {
-	return iter.data[iter.start:iter.pos]
-}
-
-// Start returns the byte position of the current token in the original data.
-func (iter *Iterator[T]) Start() int {
-	return iter.start
-}
-
-// End returns the byte position after the current token in the original data.
-func (iter *Iterator[T]) End() int {
-	return iter.pos
-}
-
-// Reset resets the iterator to the beginning of the data.
-func (iter *Iterator[T]) Reset() {
-	iter.start = 0
-	iter.pos = 0
-}
@@ -21,10 +21,9 @@ github.com/cenkalti/backoff/v5
 # github.com/cespare/xxhash/v2 v2.3.0
 ## explicit; go 1.11
 github.com/cespare/xxhash/v2
-# github.com/clipperhouse/uax29/v2 v2.2.0
+# github.com/clipperhouse/uax29/v2 v2.7.0
 ## explicit; go 1.18
 github.com/clipperhouse/uax29/v2/graphemes
-github.com/clipperhouse/uax29/v2/internal/iterators
 # github.com/containerd/errdefs v1.0.0
 ## explicit; go 1.20
 github.com/containerd/errdefs