Bugfix: Use \u001F (ASCII unit separator) as the separator between feature and label in model json file, instead of "|" because "|" may be part of a feature (in weird cases). Update model.

Author: strangetom

README.md

@@ -58,10 +58,10 @@ The model has the following accuracy on a test data set of 20% of the total data
 ╒══════════════════════════╤══════════════════════════╕
 │ Sentence-level results   │ Word-level results       │
 ╞══════════════════════════╪══════════════════════════╡
-│ Accuracy: 95.62%         │ Accuracy: 98.26%         │
-│                          │ Precision (micro) 98.25% │
-│                          │ Recall (micro) 98.26%    │
-│                          │ F1 score (micro) 98.25%  │
+│ Accuracy: 95.59%         │ Accuracy: 98.25%         │
+│                          │ Precision (micro) 98.24% │
+│                          │ Recall (micro) 98.25%    │
+│                          │ F1 score (micro) 98.24%  │
 ╘══════════════════════════╧══════════════════════════╛
 ```
 

ingredient_parser/en/data/ModelCard.en.md

@@ -130,7 +130,7 @@ The model has the following performance metrics:
 
 | Word level accuracy | Sentence level accuracy |
 | ------------------- | ----------------------- |
-| 98.26 ± 0.22%       | 95.62 ± 0.48%           |
+| 98.25 ± 0.22%       | 95.59 ± 0.48%           |
 
 These metrics were determined by executing 25 training/evaluation cycles and calculating the mean and standard deviation for the two metrics across all cycles. The uncertainty values provided represent the 99.7% confidence bounds (i.e. 3x standard deviation). The uncertainty is due to the randomisation of the selection of training and evaluation data whenever the model is trained.
 

ingredient_parser/en/data/model.en.json

@@ -11,9 +11,9 @@
 	"delta": 5e-5,
 	"linesearch": "MoreThuente",
 	"max_linesearch": 5,
-	"datetime": "2026-05-25T15:05:24.991451",
+	"datetime": "2026-07-03T21:14:34.882449",
 	"stopping_reason": "L-BFGS terminated with the stopping criteria",
 	"quantize_bits": 16,
 	"min_abs_weight": null,
-	"sha256": "334f14a6a6d39c6eb1076b61091a9b2216567b2525e2991d5f8d1bd90e250bc2"
+	"sha256": "5cb2c3d123d88611a3709eec074b146c8c04c1ea07623554e907f635a0fcf042"
 }

ingredient_parser/en/data/model.en.json.gz

@@ -421,7 +421,7 @@ def __init__(
         # weights.
         self.emission_weights = np.zeros((self.n_features, self.n_labels), dtype=dtype)
         for feat, weight in feature_weights.items():
-            feature, label = feat.split("|")
+            feature, label = feat.split("\u001f")
             feature_idx = self.features_to_idx[feature]
             label_idx = self.label_to_idx[label]
             self.emission_weights[feature_idx, label_idx] = weight
@@ -430,7 +430,7 @@ def __init__(
         # weights.
         self.transition_weights = np.zeros((self.n_labels, self.n_labels), dtype=dtype)
         for feat, weight in transition_weights.items():
-            prev_label, current_label = feat.split("|")
+            prev_label, current_label = feat.split("\u001f")
             prev_label_idx = self.label_to_idx[prev_label]
             current_label_idx = self.label_to_idx[current_label]
             self.transition_weights[prev_label_idx, current_label_idx] = weight

ingredient_parser/inference.py

@@ -35,7 +35,7 @@ def export_crfsuite_to_json(
       Name of each label (key) and their index (value).
     * state_features :  dict[str, float]
       The weights for each feature for each label.
-      The keys are the feature name and label joined with |.
+      The keys are the feature name and label joined with \u001f (ASCII unit separator).
       For example: bias:|QTY, where "bias:" is the feature name and "QTY" is the label.
     * transitions
       The weights for each label to label transition.
@@ -59,8 +59,8 @@ def export_crfsuite_to_json(
     params = CRFModelParameters(
         attributes={k: int(v) for k, v in info.attributes.items()},
         labels={k: int(v) for k, v in info.labels.items()},
-        state_features={k[0] + "|" + k[1]: v for k, v in info.state_features.items()},
-        transitions={k[0] + "|" + k[1]: v for k, v in info.transitions.items()},
+        state_features={"\u001f".join(k): v for k, v in info.state_features.items()},
+        transitions={"\u001f".join(k): v for k, v in info.transitions.items()},
         quantization_scale=1.0,
         quantization_zero_offset=0,
     )