From b0db142160ba4599a42ee77db432577da06087b5 Mon Sep 17 00:00:00 2001 From: Joel Date: Sat, 26 Sep 2026 21:59:16 +0530 Subject: [PATCH 1/4] FIX reject empty targets in exact text matching --- pyrit/analytics/text_matching.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyrit/analytics/text_matching.py b/pyrit/analytics/text_matching.py index 620d3d50fc..4346c60d76 100644 --- a/pyrit/analytics/text_matching.py +++ b/pyrit/analytics/text_matching.py @@ -67,6 +67,8 @@ def is_match(self, *, target: str, text: str) -> bool: if self._ignore_whitespace: target = target.strip() text = text.strip() + if not target: + return False if self._case_sensitive: return target in text return target.lower() in text.lower() From 32b79156b96a7d20b56283f9bf9608a6f4674a6a Mon Sep 17 00:00:00 2001 From: Joel Date: Sat, 26 Sep 2026 21:59:23 +0530 Subject: [PATCH 2/4] TEST cover empty exact-match targets --- tests/unit/analytics/test_text_matching.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/analytics/test_text_matching.py b/tests/unit/analytics/test_text_matching.py index b9a4097746..bcb097a05e 100644 --- a/tests/unit/analytics/test_text_matching.py +++ b/tests/unit/analytics/test_text_matching.py @@ -27,6 +27,11 @@ def test_empty_text(self): matcher = ExactTextMatching(case_sensitive=False) assert matcher.is_match(target="hello", text="") is False + def test_empty_target(self): + matcher = ExactTextMatching() + assert matcher.is_match(target="", text="hello world") is False + assert matcher.is_match(target=" \n ", text="hello world") is False + def test_partial_match(self): matcher = ExactTextMatching(case_sensitive=False) assert matcher.is_match(target="World", text="Hello World") is True From c40d1536d723ef73e91c43f252ec6f74a0f778fe Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 28 Sep 2026 08:46:06 +0530 Subject: [PATCH 3/4] fix: reject whitespace-only exact match targets --- pyrit/analytics/text_matching.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pyrit/analytics/text_matching.py b/pyrit/analytics/text_matching.py index 4346c60d76..bc1c89620f 100644 --- a/pyrit/analytics/text_matching.py +++ b/pyrit/analytics/text_matching.py @@ -64,11 +64,11 @@ def is_match(self, *, target: str, text: str) -> bool: """ if not text: return False + if not target.strip(): + return False if self._ignore_whitespace: target = target.strip() text = text.strip() - if not target: - return False if self._case_sensitive: return target in text return target.lower() in text.lower() @@ -121,27 +121,22 @@ def _calculate_ngram_overlap(self, *, target: str, text: str) -> float: Returns: float: A score between 0.0 and 1.0 indicating the proportion of target n-grams - found in the text. + found in the text. """ if not text: return 0.0 if len(target) < self._n: - return 0.0 # Confidence is too low for short targets + return 0.0 target_str = target if self._case_sensitive else target.lower() text_str = text if self._case_sensitive else text.lower() - # Generate all n-grams from target target_ngrams = {target_str[i : i + self._n] for i in range(len(target_str) - (self._n - 1))} - # Safety check: if no n-grams were generated, return 0.0 if not target_ngrams: return 0.0 - # Count how many target n-grams are found in text matching_ngrams = sum(int(ngram in text_str) for ngram in target_ngrams) - - # Calculate proportion of matching n-grams return matching_ngrams / len(target_ngrams) def get_overlap_score(self, *, target: str, text: str) -> float: @@ -151,10 +146,10 @@ def get_overlap_score(self, *, target: str, text: str) -> float: Useful for getting detailed scoring information. Args: - target (str): The target string to match. + target (str): The string to search for. text (str): The text to search in. Returns: - float: The n-gram overlap score between 0.0 and 1.0. + float: The n-gram overlap score between target and text. """ return self._calculate_ngram_overlap(target=target, text=text) From 56bfff6d7edb84c35313c33212a6e90b24b3e0a0 Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 28 Sep 2026 08:46:56 +0530 Subject: [PATCH 4/4] test: cover empty converted value in decoding scorer --- tests/unit/analytics/test_text_matching.py | 5 +++ tests/unit/score/test_decoding_scorer.py | 42 +++++++++++++++------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/tests/unit/analytics/test_text_matching.py b/tests/unit/analytics/test_text_matching.py index bcb097a05e..ae19f26065 100644 --- a/tests/unit/analytics/test_text_matching.py +++ b/tests/unit/analytics/test_text_matching.py @@ -32,6 +32,11 @@ def test_empty_target(self): assert matcher.is_match(target="", text="hello world") is False assert matcher.is_match(target=" \n ", text="hello world") is False + def test_whitespace_only_target_when_whitespace_is_not_ignored(self): + matcher = ExactTextMatching(ignore_whitespace=False) + assert matcher.is_match(target=" ", text="hello world") is False + assert matcher.is_match(target=" \n\t ", text="hello world") is False + def test_partial_match(self): matcher = ExactTextMatching(case_sensitive=False) assert matcher.is_match(target="World", text="Hello World") is True diff --git a/tests/unit/score/test_decoding_scorer.py b/tests/unit/score/test_decoding_scorer.py index 2faed98e6a..2f9660e5c4 100644 --- a/tests/unit/score/test_decoding_scorer.py +++ b/tests/unit/score/test_decoding_scorer.py @@ -34,8 +34,6 @@ def sample_message_pieces(): class TestDecodingScorer: async def test_decoding_scorer_original_value_match(self, patch_central_database, sample_message_pieces): user_piece, assistant_piece = sample_message_pieces - - # Mock memory to return the user request memory = MagicMock(MemoryInterface) user_message = Message(message_pieces=[user_piece]) memory.get_request_from_response.return_value = user_message @@ -45,7 +43,7 @@ async def test_decoding_scorer_original_value_match(self, patch_central_database score = await scorer._score_piece_async(assistant_piece) assert len(score) == 1 - assert score[0].get_value() is True # "secret" is in the response + assert score[0].get_value() is True assert score[0].score_type == "true_false" assert score[0].score_category == ["decoding"] @@ -62,7 +60,7 @@ async def test_decoding_scorer_converted_value_match(self, patch_central_databas score = await scorer._score_piece_async(assistant_piece) assert len(score) == 1 - assert score[0].get_value() is True # "encoded_secret" is in the response + assert score[0].get_value() is True async def test_decoding_scorer_metadata_match(self, patch_central_database, sample_message_pieces): user_piece, assistant_piece = sample_message_pieces @@ -77,7 +75,7 @@ async def test_decoding_scorer_metadata_match(self, patch_central_database, samp score = await scorer._score_piece_async(assistant_piece) assert len(score) == 1 - assert score[0].get_value() is True # "decoded_secret" from metadata is in response + assert score[0].get_value() is True async def test_decoding_scorer_no_match(self, patch_central_database, sample_message_pieces): user_piece, assistant_piece = sample_message_pieces @@ -94,6 +92,30 @@ async def test_decoding_scorer_no_match(self, patch_central_database, sample_mes assert len(score) == 1 assert score[0].get_value() is False + async def test_decoding_scorer_empty_converted_value_does_not_match( + self, patch_central_database, sample_message_pieces + ): + user_piece, _ = sample_message_pieces + user_piece.original_value = "secret" + user_piece.converted_value = "" + + assistant_piece = MessagePiece( + role="assistant", + original_value="I refuse", + converted_value="I refuse", + ) + + memory = MagicMock(MemoryInterface) + user_message = Message(message_pieces=[user_piece]) + memory.get_request_from_response.return_value = user_message + + with patch.object(CentralMemory, "get_memory_instance", return_value=memory): + scorer = DecodingScorer(categories=["decoding"]) + score = await scorer._score_piece_async(assistant_piece) + + assert len(score) == 1 + assert score[0].get_value() is False + async def test_decoding_scorer_case_insensitive(self, patch_central_database, sample_message_pieces): user_piece, assistant_piece = sample_message_pieces assistant_piece.converted_value = "Response with SECRET in it" @@ -103,12 +125,11 @@ async def test_decoding_scorer_case_insensitive(self, patch_central_database, sa memory.get_request_from_response.return_value = user_message with patch.object(CentralMemory, "get_memory_instance", return_value=memory): - # Default is case insensitive scorer = DecodingScorer(categories=["decoding"]) score = await scorer._score_piece_async(assistant_piece) assert len(score) == 1 - assert score[0].get_value() is True # Case insensitive match + assert score[0].get_value() is True async def test_decoding_scorer_case_sensitive(self, patch_central_database, sample_message_pieces): user_piece, assistant_piece = sample_message_pieces @@ -124,11 +145,10 @@ async def test_decoding_scorer_case_sensitive(self, patch_central_database, samp score = await scorer._score_piece_async(assistant_piece) assert len(score) == 1 - assert score[0].get_value() is False # Case sensitive, no match + assert score[0].get_value() is False async def test_decoding_scorer_approximate_matching(self, patch_central_database, sample_message_pieces): user_piece, assistant_piece = sample_message_pieces - # Partial match - has some n-grams in common assistant_piece.converted_value = "Response with sec ret characters separated" memory = MagicMock(MemoryInterface) @@ -136,13 +156,11 @@ async def test_decoding_scorer_approximate_matching(self, patch_central_database memory.get_request_from_response.return_value = user_message with patch.object(CentralMemory, "get_memory_instance", return_value=memory): - # Use approximate matching with low threshold text_matcher = ApproximateTextMatching(threshold=0.3, n=3, case_sensitive=False) scorer = DecodingScorer(text_matcher=text_matcher, categories=["decoding"]) score = await scorer._score_piece_async(assistant_piece) assert len(score) == 1 - # Should detect partial match with low threshold async def test_decoding_scorer_approximate_no_match(self, patch_central_database, sample_message_pieces): user_piece, assistant_piece = sample_message_pieces @@ -158,4 +176,4 @@ async def test_decoding_scorer_approximate_no_match(self, patch_central_database score = await scorer._score_piece_async(assistant_piece) assert len(score) == 1 - assert score[0].get_value() is False # Below threshold + assert score[0].get_value() is False