Merge branch 'dev'

Author: umarbutler

CHANGELOG.md

@@ -1,6 +1,11 @@
 ## Changelog 🔄
 All notable changes to semchunk will be documented here. This project adheres to [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [4.1.1] - 2026-06-13
+### Changed
+- Sped up chunking, particularly of real-world documents at typical chunk sizes, by only ever materializing the comparatively rare multi-character whitespace runs when identifying the most structurally meaningful splitter, sparing `_split_text()` from materializing the far more numerous single-character runs (such as the single spaces between words) that can never be the longest run unless no longer run exists.
+- Further sped up chunking by fusing the construction of splits' start offsets into a single pass and by iterating directly from the start of one chunk to the next rather than visiting, and skipping over, every split already merged into a chunk.
+
 ## [4.1.0] - 2026-06-12
 ### Changed
 - Replaced [`mpire`](https://github.com/sybrenjansen/mpire) with the standard library's `multiprocessing` module for chunking multiple texts with multiple processes. On systems that support forking processes (namely, POSIX systems such as Linux and macOS), this makes [`tqdm`](https://github.com/tqdm/tqdm) semchunk's only dependency. On systems that do not support forking processes (namely, Windows), [`dill`](https://github.com/uqfoundation/dill) is now used to serialize the chunk function for spawned worker processes (as the standard library's `pickle` cannot serialize unpicklable token counters such as closures and lambdas), and so `dill` is now a Windows-only dependency, replacing the heavier `mpire`, which had depended on it.
@@ -191,6 +196,7 @@ All notable changes to semchunk will be documented here. This project adheres to
 ### Added
 - Added the `chunk()` function, which splits text into semantically meaningful chunks of a specified size as determined by a provided token counter.
 
+[4.1.1]: https://github.com/isaacus-dev/semchunk/compare/v4.1.0...v4.1.1
 [4.1.0]: https://github.com/isaacus-dev/semchunk/compare/v4.0.0...v4.1.0
 [4.0.0]: https://github.com/isaacus-dev/semchunk/compare/v3.2.5...v4.0.0
 [3.2.5]: https://github.com/isaacus-dev/semchunk/compare/v3.2.4...v3.2.5

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "semchunk"
-version = "4.1.0"
+version = "4.1.1"
 authors = [
     {name="Isaacus", email="hello@isaacus.com"},
     {name="Umar Butler", email="umar@isaacus.com"},

src/semchunk/semchunk.py

@@ -72,6 +72,18 @@
     re.escape(splitter) for splitter in _NON_WHITESPACE_STRUCTURAL_SPLITTERS
 )
 
+# NOTE Whitespace runs are matched with a `{2,}` quantifier rather than a `+` quantifier so that we only ever need the comparatively rare multi-character runs to find the longest run, sparing us from materializing the far more numerous single-character runs (e.g., the single spaces between words), which are never the longest run unless no multi-character run exists.
+_NEWLINE_RUNS = re.compile(r"[\r\n]{2,}")
+_TAB_RUNS = re.compile(r"\t{2,}")
+_WHITESPACE_RUNS = re.compile(r"\s{2,}")
+_NEWLINE_CHAR = re.compile(r"[\r\n]")
+_WHITESPACE_CHAR = re.compile(r"\s")
+_PRECEDER_WHITESPACE_PATTERNS = tuple(
+    (escaped_preceder, re.compile(rf"{escaped_preceder}(\s)"))
+    for escaped_preceder in _REGEX_ESCAPED_NON_WHITESPACE_SEMANTIC_SPLITTERS
+)
+"""A tuple of `(escaped_preceder, compiled_pattern)` pairs used to target whitespace characters preceded by structurally meaningful non-whitespace splitters."""
+
 
 def _split_text(text: str) -> tuple[str, bool, list[str]]:
     """Split text using the most structurally meaningful splitter possible."""
@@ -84,18 +96,25 @@ def _split_text(text: str) -> tuple[str, bool, list[str]]:
     # - the largest sequence of whitespace characters or, if the largest such sequence is only a single character and there exists a whitespace character preceded by a structurally meaningful non-whitespace splitter, then that whitespace character; and
     # - a structurally meaningful non-whitespace splitter.
     if "\n" in text or "\r" in text:
-        splitter = max(re.findall(r"[\r\n]+", text), key=len)
+        runs = _NEWLINE_RUNS.findall(text)
+        splitter = max(runs, key=len) if runs else _NEWLINE_CHAR.search(text).group()
 
     elif "\t" in text:
-        splitter = max(re.findall(r"\t+", text), key=len)
+        runs = _TAB_RUNS.findall(text)
+        splitter = max(runs, key=len) if runs else "\t"
+
+    elif whitespace := _WHITESPACE_CHAR.search(text):
+        runs = _WHITESPACE_RUNS.findall(text)
 
-    elif re.search(r"\s", text):
-        splitter = max(re.findall(r"\s+", text), key=len)
+        if runs:
+            splitter = max(runs, key=len)
 
         # If the splitter is only a single character, see if we can target whitespace characters that are preceded by structurally meaningful non-whitespace splitters to avoid splitting in the middle of sentences.
-        if len(splitter) == 1:
-            for escaped_preceder in _REGEX_ESCAPED_NON_WHITESPACE_SEMANTIC_SPLITTERS:
-                if whitespace_preceded_by_preceder := re.search(rf"{escaped_preceder}(\s)", text):
+        else:
+            splitter = whitespace.group()
+
+            for escaped_preceder, preceder_pattern in _PRECEDER_WHITESPACE_PATTERNS:
+                if whitespace_preceded_by_preceder := preceder_pattern.search(text):
                     splitter = whitespace_preceded_by_preceder.group(1)
                     escaped_splitter = re.escape(splitter)
 
@@ -493,20 +512,20 @@ def chunk(
 
     offsets: list = []
     splitter_len = len(splitter)
-    split_lens = [len(split) for split in splits]
-    local_split_starts = list(accumulate([0] + [split_len + splitter_len for split_len in split_lens]))
-    split_starts = [start + _start for start in local_split_starts]
-    num_splits_plus_one = len(splits) + 1
+    local_split_starts = list(accumulate([len(split) + splitter_len for split in splits], initial=0))
+    num_splits = len(splits)
+    num_splits_plus_one = num_splits + 1
 
     chunks = []
-    skips = set()
-    """A set of indices of splits to skip because they have already been added to a chunk."""
 
-    # Iterate through the splits.
-    for i, (split, split_start) in enumerate(zip(splits, split_starts)):
-        # Skip the split if it has already been added to a chunk.
-        if i in skips:
-            continue
+    # Iterate through the splits, jumping straight to the start of each new chunk so that splits already merged into a chunk are never revisited.
+    i = 0
+
+    while i < num_splits:
+        split = splits[i]
+
+        # Compute the start offset of the split in the original text. NOTE `split_starts[i]` is `local_split_starts[i] + _start`.
+        split_start = local_split_starts[i] + _start
 
         # If the split is over the chunk size, recursively chunk it.
         if token_counter(split) > local_chunk_size:
@@ -522,10 +541,13 @@ def chunk(
             chunks.extend(new_chunks)
             offsets.extend(new_offsets)
 
+            # The next chunk starts at the very next split.
+            next_i = i + 1
+
         # If the split is equal to or under the chunk size, add it and any subsequent splits to a new chunk until the chunk size is reached.
         else:
-            # Merge the split with subsequent splits until the chunk size is reached.
-            final_split_in_chunk_i, new_chunk = merge_splits(
+            # Merge the split with subsequent splits until the chunk size is reached. NOTE `next_i` is the index of the first split not included in the merged chunk and so the start of the next chunk.
+            next_i, new_chunk = merge_splits(
                 text=text,
                 split_starts=local_split_starts,
                 splitter_len=splitter_len,
@@ -536,20 +558,15 @@ def chunk(
                 high=num_splits_plus_one,
             )
 
-            # Mark any splits included in the new chunk for exclusion from future chunks.
-            skips.update(range(i + 1, final_split_in_chunk_i))
-
             # Add the chunk.
             chunks.append(new_chunk)
 
             # Add the chunk's offsets.
-            split_end = split_starts[final_split_in_chunk_i] - splitter_len
+            split_end = local_split_starts[next_i] + _start - splitter_len
             offsets.append((split_start, split_end))
 
-        # If the splitter is not whitespace and the split is not the last split, add the splitter to the end of the latest chunk if doing so would not cause it to exceed the chunk size otherwise add the splitter as a new chunk.
-        if not splitter_is_whitespace and not (
-            i == len(splits) - 1 or all(j in skips for j in range(i + 1, len(splits)))
-        ):
+        # If the splitter is not whitespace and the chunk just added is not the last chunk, add the splitter to the end of the latest chunk if doing so would not cause it to exceed the chunk size otherwise add the splitter as a new chunk.
+        if not splitter_is_whitespace and next_i < num_splits:
             if token_counter(last_chunk_with_splitter := chunks[-1] + splitter) <= local_chunk_size:
                 chunks[-1] = last_chunk_with_splitter
                 start, end = offsets[-1]
@@ -561,6 +578,9 @@ def chunk(
                 chunks.append(splitter)
                 offsets.append((start, start + splitter_len))
 
+        # Advance to the start of the next chunk.
+        i = next_i
+
     # If this is the first call, remove any empty chunks as well as chunks comprised entirely of whitespace and then overlap the chunks if desired and finally return the chunks, optionally with their offsets.
     if is_first_call:
         # Remove empty chunks.