From 2fc6604fe35ad2461f9cbdfbb52f8f525b06f54f Mon Sep 17 00:00:00 2001
From: stephantul <stephantul@gmail.com>
Date: Sun, 18 Oct 2026 18:27:42 +0200
Subject: [PATCH 1/2] fix: decode tokens to avoid re-encoding them

---
 model2vec/tokenizer/tokenizer.py |  4 +++-
 tests/test_distillation.py       | 17 ++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/model2vec/tokenizer/tokenizer.py b/model2vec/tokenizer/tokenizer.py
index c1cb194..f1d5751 100644
--- a/model2vec/tokenizer/tokenizer.py
+++ b/model2vec/tokenizer/tokenizer.py
@@ -135,6 +135,8 @@ def turn_tokens_into_ids(tokens: list[str], model: TokenizerModel) -> list[list[
         if token_id is not None:
             token_ids.append([*prefix, token_id, *suffix])
         else:
-            token_ids.append(tokenizer.encode(token).ids)
+            decoded = model.preprocessor.decode(token)
+            enc = tokenizer.encode(decoded.decoded).ids
+            token_ids.append(enc)
 
     return token_ids
diff --git a/tests/test_distillation.py b/tests/test_distillation.py
index ef5b1cd..07894b9 100644
--- a/tests/test_distillation.py
+++ b/tests/test_distillation.py
@@ -21,7 +21,7 @@
 from model2vec.distill.distillation import distill, distill_from_model
 from model2vec.distill.inference import PoolingMode, apply_pca, compute_weights, create_embeddings
 from model2vec.model import StaticModel
-from model2vec.tokenizer import clean_and_create_vocabulary
+from model2vec.tokenizer import clean_and_create_vocabulary, turn_tokens_into_ids
 
 try:
     # For huggingface_hub>=0.25.0
@@ -503,6 +503,21 @@ def test_clean_and_create_vocabulary(
             assert any(expected_warning in logged_warning for logged_warning in logged_warnings)
 
 
+@pytest.mark.parametrize(
+    "token,expected_tokens",
+    [
+        ("##ing", ["[CLS]", "##ing", "[SEP]"]),
+        ("##xyzq", ["[CLS]", "x", "##y", "##z", "##q", "[SEP]"]),
+        ("hello world", ["[CLS]", "hello", "world", "[SEP]"]),
+    ],
+)
+def test_turn_tokens_into_ids(mock_tokenizermodel: TokenizerModel, token: str, expected_tokens: list[str]) -> None:
+    """Test that tokens are decoded before being re-encoded."""
+    ids = turn_tokens_into_ids([token], mock_tokenizermodel)
+    expected_ids = [mock_tokenizermodel.vocabulary[t] for t in expected_tokens]
+    assert ids == [expected_ids]
+
+
 @pytest.mark.parametrize(
     "pooling,with_pooler,expected_rows",
     [

From 8d131ee3852391225fb1ea5128bd3066f4313c4a Mon Sep 17 00:00:00 2001
From: stephantul <stephantul@gmail.com>
Date: Sun, 18 Oct 2026 18:36:33 +0200
Subject: [PATCH 2/2] update tests and baselines

---
 ...ba-NLP___gte-modernbert-base_baseline.json | 20 +++++++++----------
 tests/test_distillation.py                    | 14 +++++++++++++
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/tests/integration/data/Alibaba-NLP___gte-modernbert-base_baseline.json b/tests/integration/data/Alibaba-NLP___gte-modernbert-base_baseline.json
index 165b811..54905f9 100644
--- a/tests/integration/data/Alibaba-NLP___gte-modernbert-base_baseline.json
+++ b/tests/integration/data/Alibaba-NLP___gte-modernbert-base_baseline.json
@@ -3,10 +3,10 @@
   "configs": {
     "custom_vocab": {
       "embedding_dim": 32,
-      "embedding_mean": 0.003367,
+      "embedding_mean": 0.003366,
       "embedding_rank": 32,
       "embedding_rows": 50269,
-      "embedding_std": 1.80662,
+      "embedding_std": 1.806562,
       "first_tokens": [
         "!",
         "\"",
@@ -33,14 +33,14 @@
         "\u0120flimzycr"
       ],
       "mteb_sts_scores": {
-        "BIOSSES": 0.51453,
-        "SICK-R": 0.540092,
-        "STS12": 0.566079,
-        "STS13": 0.630178,
-        "STS14": 0.548316,
-        "STS15": 0.613225,
-        "STS16": 0.565322,
-        "STSBenchmark": 0.541931
+        "BIOSSES": 0.514596,
+        "SICK-R": 0.540098,
+        "STS12": 0.566178,
+        "STS13": 0.630239,
+        "STS14": 0.548269,
+        "STS15": 0.613244,
+        "STS16": 0.565316,
+        "STSBenchmark": 0.541872
       },
       "token_order_hash": "c2f9a27e98ef519d36c74cbde4ccc7ab023a23896aac6351f30c41c900daac78"
     },
diff --git a/tests/test_distillation.py b/tests/test_distillation.py
index 07894b9..3dd195f 100644
--- a/tests/test_distillation.py
+++ b/tests/test_distillation.py
@@ -518,6 +518,20 @@ def test_turn_tokens_into_ids(mock_tokenizermodel: TokenizerModel, token: str, e
     assert ids == [expected_ids]
 
 
+def test_turn_tokens_into_ids_byte_level() -> None:
+    """Test that unmatched byte-level tokens are decoded to text with a leading space before being re-encoded."""
+    vocab = ["[UNK]", "Ġ", "Ä", "ł", "w", "o", "r", "d", "1"]
+    tokenizer = Tokenizer(BPE(vocab={token: idx for idx, token in enumerate(vocab)}, merges=[], unk_token="[UNK]"))
+    tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)  # type: ignore
+    model = TokenizerModel.from_transformers_tokenizer(PreTrainedTokenizerFast(tokenizer_object=tokenizer))
+    assert "Ġword1" not in model.vocabulary
+
+    ids = turn_tokens_into_ids(["Ġword1"], model)
+    expected_ids = model.to_tokenizer().encode(" word1").ids
+    assert expected_ids == [model.vocabulary[t] for t in ["Ġ", "w", "o", "r", "d", "1"]]
+    assert ids == [expected_ids]
+
+
 @pytest.mark.parametrize(
     "pooling,with_pooler,expected_rows",
     [