From cdcdc9cc835b2fb4dd8361d90d039da3420cb1e2 Mon Sep 17 00:00:00 2001 From: stephantul Date: Sun, 20 Sep 2026 21:11:38 +0200 Subject: [PATCH 1/4] fix: disable padding during distill and inference --- model2vec/model.py | 2 ++ model2vec/tokenizer/tokenizer.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/model2vec/model.py b/model2vec/model.py index a13ab46..9aab5a5 100644 --- a/model2vec/model.py +++ b/model2vec/model.py @@ -80,6 +80,8 @@ def __init__( self.token_mapping: np.ndarray | None = token_mapping self.tokenizer = copy.deepcopy(tokenizer) + # Turn off padding. + self.tokenizer.no_padding() self.unk_token_id = _get_unk_token_id(self.tokenizer) self.median_token_length = int(np.median([len(token) for token in self.tokens])) diff --git a/model2vec/tokenizer/tokenizer.py b/model2vec/tokenizer/tokenizer.py index 62b5623..2d26672 100644 --- a/model2vec/tokenizer/tokenizer.py +++ b/model2vec/tokenizer/tokenizer.py @@ -5,10 +5,21 @@ from typing import Sequence from skeletoken import TokenizerModel +from skeletoken.padding import Padding logger = logging.getLogger(__name__) +def _replace_padding(model: TokenizerModel) -> TokenizerModel: + """Replaces the model's padding module with a benign one if it isn't already.""" + padding = model.padding + # If the model has a padding strategy that is not the standard one + if padding is not None: + model.padding = Padding(pad_id=padding.pad_id, pad_token=padding.pad_token, pad_type_id=0) + + return model + + def clean_and_create_vocabulary( model: TokenizerModel, vocabulary_to_add: Sequence[str], @@ -33,6 +44,7 @@ def clean_and_create_vocabulary( # Remove the post processor. model.post_processor = None + model = _replace_padding(model) internal_tokens: list[str] = model.sorted_vocabulary if token_remove_regex: From b3fd011e69d5307e080f001bd0e5e8ecd5909085 Mon Sep 17 00:00:00 2001 From: stephantul Date: Sun, 20 Sep 2026 21:20:41 +0200 Subject: [PATCH 2/4] clarify --- model2vec/tokenizer/tokenizer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model2vec/tokenizer/tokenizer.py b/model2vec/tokenizer/tokenizer.py index 2d26672..c1cb194 100644 --- a/model2vec/tokenizer/tokenizer.py +++ b/model2vec/tokenizer/tokenizer.py @@ -11,9 +11,8 @@ def _replace_padding(model: TokenizerModel) -> TokenizerModel: - """Replaces the model's padding module with a benign one if it isn't already.""" + """Replaces the model's padding module with a benign one.""" padding = model.padding - # If the model has a padding strategy that is not the standard one if padding is not None: model.padding = Padding(pad_id=padding.pad_id, pad_token=padding.pad_token, pad_type_id=0) From 3273d244ee23b76a360caa3327d72615fb9c769b Mon Sep 17 00:00:00 2001 From: stephantul Date: Sun, 20 Sep 2026 21:34:05 +0200 Subject: [PATCH 3/4] remove padding in a way that sustains the token --- model2vec/model.py | 7 +++++-- tests/test_trainable.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/model2vec/model.py b/model2vec/model.py index 9aab5a5..a55dfdb 100644 --- a/model2vec/model.py +++ b/model2vec/model.py @@ -80,8 +80,11 @@ def __init__( self.token_mapping: np.ndarray | None = token_mapping self.tokenizer = copy.deepcopy(tokenizer) - # Turn off padding. - self.tokenizer.no_padding() + padding = self.tokenizer.padding + if padding is not None: + self.tokenizer.enable_padding( + pad_id=padding["pad_id"], pad_token=padding["pad_token"], pad_type_id=padding["pad_type_id"], length=0 + ) self.unk_token_id = _get_unk_token_id(self.tokenizer) self.median_token_length = int(np.median([len(token) for token in self.tokens])) diff --git a/tests/test_trainable.py b/tests/test_trainable.py index 2bc744a..2929deb 100644 --- a/tests/test_trainable.py +++ b/tests/test_trainable.py @@ -642,6 +642,20 @@ def test_get_probable_pad_token_id(mock_tokenizer: Tokenizer, caplog: pytest.Log assert "No known pad token found, using 0 as default" in caplog.text +def test_get_probable_pad_token_id_through_static_model(mock_vectors: np.ndarray, mock_tokenizer: Tokenizer) -> None: + """Test that a non-standard pad token survives StaticModel construction.""" + tokenizer_model = TokenizerModel.from_tokenizer(mock_tokenizer) + tokenizer_model.pad_token = "word1" + pad_id = tokenizer_model.pad_token_id + assert pad_id != 0 + + t = tokenizer_model.to_tokenizer() + model = StaticModel(vectors=mock_vectors, tokenizer=t) + + assert model.tokenizer.padding is not None + assert get_probable_pad_token_id(model.tokenizer) == pad_id + + def test_determine_class_weight(mock_trained_pipeline: StaticModelForClassification) -> None: """Test what the class weights are.""" w_dict = dict(zip(mock_trained_pipeline.classes, [0.5, 3])) From f4cc9d3981c523c88b083f225a67be7cf1f67992 Mon Sep 17 00:00:00 2001 From: stephantul Date: Sun, 20 Sep 2026 21:57:54 +0200 Subject: [PATCH 4/4] update minilm baseline --- ...nsformers___all-MiniLM-L6-v2_baseline.json | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/integration/data/sentence-transformers___all-MiniLM-L6-v2_baseline.json b/tests/integration/data/sentence-transformers___all-MiniLM-L6-v2_baseline.json index 9a7be03..7db8c18 100644 --- a/tests/integration/data/sentence-transformers___all-MiniLM-L6-v2_baseline.json +++ b/tests/integration/data/sentence-transformers___all-MiniLM-L6-v2_baseline.json @@ -33,14 +33,14 @@ "flimzycrag" ], "mteb_sts_scores": { - "BIOSSES": 0.500132, - "SICK-R": 0.565144, - "STS12": 0.594863, - "STS13": 0.653794, - "STS14": 0.596517, - "STS15": 0.611504, - "STS16": 0.569703, - "STSBenchmark": 0.559642 + "BIOSSES": 0.497391, + "SICK-R": 0.567003, + "STS12": 0.590715, + "STS13": 0.663817, + "STS14": 0.600884, + "STS15": 0.614552, + "STS16": 0.574441, + "STSBenchmark": 0.565802 }, "token_order_hash": "09d9aaaedc7d41b2fd1e3ffa64f16794a2cefe9ffc7faf1055ae5d49350c3060" }, @@ -76,14 +76,14 @@ "##\uff5e" ], "mteb_sts_scores": { - "BIOSSES": 0.656286, - "SICK-R": 0.611602, - "STS12": 0.620366, - "STS13": 0.735474, - "STS14": 0.670425, - "STS15": 0.745883, - "STS16": 0.67784, - "STSBenchmark": 0.655636 + "BIOSSES": 0.6499, + "SICK-R": 0.612639, + "STS12": 0.617488, + "STS13": 0.739954, + "STS14": 0.672889, + "STS15": 0.747828, + "STS16": 0.68201, + "STSBenchmark": 0.660814 }, "token_order_hash": "d881fab650dd73240615f87f531ff31a6135f961933ebc32c0dfdd130763e7a8" }