fix: decode tokens to avoid re-encoding them

model2vec/tokenizer/tokenizer.py

@@ -135,6 +135,8 @@ def turn_tokens_into_ids(tokens: list[str], model: TokenizerModel) -> list[list[
         if token_id is not None:
             token_ids.append([*prefix, token_id, *suffix])
         else:
-            token_ids.append(tokenizer.encode(token).ids)
+            decoded = model.preprocessor.decode(token)
+            enc = tokenizer.encode(decoded.decoded).ids
+            token_ids.append(enc)
 
     return token_ids

tests/integration/data/Alibaba-NLP___gte-modernbert-base_baseline.json

@@ -3,10 +3,10 @@
   "configs": {
     "custom_vocab": {
       "embedding_dim": 32,
-      "embedding_mean": 0.003367,
+      "embedding_mean": 0.003366,
       "embedding_rank": 32,
       "embedding_rows": 50269,
-      "embedding_std": 1.80662,
+      "embedding_std": 1.806562,
       "first_tokens": [
         "!",
         "\"",
@@ -33,14 +33,14 @@
         "\u0120flimzycr"
       ],
       "mteb_sts_scores": {
-        "BIOSSES": 0.51453,
-        "SICK-R": 0.540092,
-        "STS12": 0.566079,
-        "STS13": 0.630178,
-        "STS14": 0.548316,
-        "STS15": 0.613225,
-        "STS16": 0.565322,
-        "STSBenchmark": 0.541931
+        "BIOSSES": 0.514596,
+        "SICK-R": 0.540098,
+        "STS12": 0.566178,
+        "STS13": 0.630239,
+        "STS14": 0.548269,
+        "STS15": 0.613244,
+        "STS16": 0.565316,
+        "STSBenchmark": 0.541872
       },
       "token_order_hash": "c2f9a27e98ef519d36c74cbde4ccc7ab023a23896aac6351f30c41c900daac78"
     },

tests/test_distillation.py

@@ -21,7 +21,7 @@
 from model2vec.distill.distillation import distill, distill_from_model
 from model2vec.distill.inference import PoolingMode, apply_pca, compute_weights, create_embeddings
 from model2vec.model import StaticModel
-from model2vec.tokenizer import clean_and_create_vocabulary
+from model2vec.tokenizer import clean_and_create_vocabulary, turn_tokens_into_ids
 
 try:
     # For huggingface_hub>=0.25.0
@@ -503,6 +503,35 @@ def test_clean_and_create_vocabulary(
             assert any(expected_warning in logged_warning for logged_warning in logged_warnings)
 
 
+@pytest.mark.parametrize(
+    "token,expected_tokens",
+    [
+        ("##ing", ["[CLS]", "##ing", "[SEP]"]),
+        ("##xyzq", ["[CLS]", "x", "##y", "##z", "##q", "[SEP]"]),
+        ("hello world", ["[CLS]", "hello", "world", "[SEP]"]),
+    ],
+)
+def test_turn_tokens_into_ids(mock_tokenizermodel: TokenizerModel, token: str, expected_tokens: list[str]) -> None:
+    """Test that tokens are decoded before being re-encoded."""
+    ids = turn_tokens_into_ids([token], mock_tokenizermodel)
+    expected_ids = [mock_tokenizermodel.vocabulary[t] for t in expected_tokens]
+    assert ids == [expected_ids]
+
+
+def test_turn_tokens_into_ids_byte_level() -> None:
+    """Test that unmatched byte-level tokens are decoded to text with a leading space before being re-encoded."""
+    vocab = ["[UNK]", "Ġ", "Ä", "ł", "w", "o", "r", "d", "1"]
+    tokenizer = Tokenizer(BPE(vocab={token: idx for idx, token in enumerate(vocab)}, merges=[], unk_token="[UNK]"))
+    tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)  # type: ignore
+    model = TokenizerModel.from_transformers_tokenizer(PreTrainedTokenizerFast(tokenizer_object=tokenizer))
+    assert "Ġword1" not in model.vocabulary
+
+    ids = turn_tokens_into_ids(["Ġword1"], model)
+    expected_ids = model.to_tokenizer().encode(" word1").ids
+    assert expected_ids == [model.vocabulary[t] for t in ["Ġ", "w", "o", "r", "d", "1"]]
+    assert ids == [expected_ids]
+
+
 @pytest.mark.parametrize(
     "pooling,with_pooler,expected_rows",
     [