From 197dca79bf4dc8b69968a3ff1d47c8d2f9c88f8f Mon Sep 17 00:00:00 2001 From: gitikavj Date: Thu, 15 Oct 2026 19:48:11 +0000 Subject: [PATCH] fix(evaluation): isolate custom simulation template rendering Render custom simulation prompt strings with a dedicated Jinja2 environment while keeping bundled templates on the existing file-backed environment. Require Jinja2 3.1.6 for custom template rendering. --- pyproject.toml | 2 +- .../evaluation/runner/dataset_types.py | 3 +- .../evaluation/runner/prompts/__init__.py | 46 ++++++++++++++++--- .../test_simulated_scenario_executor.py | 34 ++++++++++++++ uv.lock | 2 +- 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2cfb9275..61db0b2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -182,7 +182,7 @@ strands-agents-evals = [ "strands-agents-evals>=1.0.3,<2.0.0" ] simulation = [ - "jinja2>=3.1.0", + "jinja2>=3.1.6", "strands-agents-evals>=1.0.3,<2.0.0", ] datasets = [ diff --git a/src/bedrock_agentcore/evaluation/runner/dataset_types.py b/src/bedrock_agentcore/evaluation/runner/dataset_types.py index 4e9f63b6..1f7bff02 100644 --- a/src/bedrock_agentcore/evaluation/runner/dataset_types.py +++ b/src/bedrock_agentcore/evaluation/runner/dataset_types.py @@ -121,7 +121,8 @@ class SimulationConfig(BaseModel): default model when None. system_prompt_template: Jinja2 system prompt template for the actor. Must contain an ``{{ actor_profile }}`` placeholder. When None, the built-in - ``structured_user_simulator.j2`` template is used. + ``structured_user_simulator.j2`` template is used. The template is rendered + in an isolated Jinja2 environment. input_type: Pydantic model class describing the agent's expected input. When set, ``input`` values in SimulatedScenario are validated into this type for the first agent call. For subsequent turns the actor is diff --git a/src/bedrock_agentcore/evaluation/runner/prompts/__init__.py b/src/bedrock_agentcore/evaluation/runner/prompts/__init__.py index 8b36bdc8..dc28d9cf 100644 --- a/src/bedrock_agentcore/evaluation/runner/prompts/__init__.py +++ b/src/bedrock_agentcore/evaluation/runner/prompts/__init__.py @@ -5,23 +5,28 @@ :func:`render_template_string` for caller-supplied template strings. """ +import re from pathlib import Path from typing import Any _PROMPTS_DIR = Path(__file__).parent _environment = None +_string_environment = None + +_JINJA2_IMPORT_ERROR = ( + "jinja2 is required for SimulatedScenario execution. Install it with: pip install 'bedrock-agentcore[simulation]'" +) +_MIN_STRING_TEMPLATE_JINJA2_VERSION = (3, 1, 6) def _get_environment(): + """Return the environment for built-in, trusted ``.j2`` files shipped with the SDK.""" global _environment if _environment is None: try: from jinja2 import Environment, FileSystemLoader except ImportError as e: - raise ImportError( - "jinja2 is required for SimulatedScenario execution. " - "Install it with: pip install 'bedrock-agentcore[simulation]'" - ) from e + raise ImportError(_JINJA2_IMPORT_ERROR) from e _environment = Environment( # nosec B701 - templates render plain text/JSON, not HTML loader=FileSystemLoader(_PROMPTS_DIR), autoescape=False, @@ -30,6 +35,32 @@ def _get_environment(): return _environment +def _get_string_environment(): + """Return an isolated environment for caller-supplied template strings. + + ``ImmutableSandboxedEnvironment`` keeps input values unchanged, while omitting a + loader keeps template strings independent from the bundled template files. + """ + global _string_environment + if _string_environment is None: + try: + import jinja2 + from jinja2.sandbox import ImmutableSandboxedEnvironment + except ImportError as e: + raise ImportError(_JINJA2_IMPORT_ERROR) from e + version = tuple(int(part) for part in re.findall(r"\d+", jinja2.__version__)[:3]) + if version < _MIN_STRING_TEMPLATE_JINJA2_VERSION: + raise ImportError( + f"jinja2>=3.1.6 is required to render system_prompt_template " + f"(found {jinja2.__version__}). Install it with: pip install 'bedrock-agentcore[simulation]'" + ) + _string_environment = ImmutableSandboxedEnvironment( # nosec B701 - templates render plain text/JSON + autoescape=False, + keep_trailing_newline=True, + ) + return _string_environment + + def render_template_file(name: str, **kwargs: Any) -> str: """Render a built-in ``.j2`` template file by name. @@ -44,7 +75,7 @@ def render_template_file(name: str, **kwargs: Any) -> str: def render_template_string(template_str: str, **kwargs: Any) -> str: - """Render a caller-supplied Jinja2 template string. + """Render a caller-supplied Jinja2 template string in a sandbox. Args: template_str: A Jinja2 template string (use ``{{ variable }}`` syntax). @@ -52,5 +83,8 @@ def render_template_string(template_str: str, **kwargs: Any) -> str: Returns: The rendered template string. + + Raises: + jinja2.exceptions.SecurityError: If the template attempts an unsafe operation. """ - return _get_environment().from_string(template_str).render(**kwargs) + return _get_string_environment().from_string(template_str).render(**kwargs) diff --git a/tests/bedrock_agentcore/evaluation/runner/test_simulated_scenario_executor.py b/tests/bedrock_agentcore/evaluation/runner/test_simulated_scenario_executor.py index e74ae257..f498b963 100644 --- a/tests/bedrock_agentcore/evaluation/runner/test_simulated_scenario_executor.py +++ b/tests/bedrock_agentcore/evaluation/runner/test_simulated_scenario_executor.py @@ -415,6 +415,40 @@ def test_render_template_string_handles_json_curly_braces(self): result = render_template_string(template, schema=schema) assert schema in result + def test_render_template_string_supports_control_flow(self): + template = "{% if scenario_description %}Scenario: {{ scenario_description }}{% endif %}" + template += "|{{ actor_profile.goal }}" + result = render_template_string(template, scenario_description="pizza", actor_profile={"goal": "order"}) + assert result == "Scenario: pizza|order" + + def test_render_template_string_restricts_private_attribute_access(self): + from jinja2.exceptions import SecurityError + + with pytest.raises(SecurityError): + render_template_string("{{ self.__init__.__globals__ }}") + + def test_render_template_string_requires_supported_jinja2(self, monkeypatch): + import jinja2 + + import bedrock_agentcore.evaluation.runner.prompts as prompts + + monkeypatch.setattr(prompts, "_string_environment", None) + monkeypatch.setattr(jinja2, "__version__", "3.1.5") + with pytest.raises(ImportError, match=r"jinja2>=3\.1\.6"): + render_template_string("{{ x }}", x=1) + + def test_render_template_string_blocks_mutation_of_inputs(self): + from jinja2.exceptions import SecurityError + + profile = {"traits": ["calm"]} + with pytest.raises(SecurityError): + render_template_string("{{ actor_profile.traits.append('focused') }}", actor_profile=profile) + assert profile == {"traits": ["calm"]} + + def test_render_template_string_cannot_include_files(self): + with pytest.raises(TypeError): + render_template_string("{% include 'structured_user_simulator.j2' %}") + def test_render_system_prompt_output_type_injected(self, mock_strands): """output_type schema must appear in the rendered prompt.""" _, _, mock_profile_cls = mock_strands diff --git a/uv.lock b/uv.lock index 0c80823e..36da76c4 100644 --- a/uv.lock +++ b/uv.lock @@ -596,7 +596,7 @@ requires-dist = [ { name = "botocore", specifier = ">=1.43.115" }, { name = "deepeval", marker = "extra == 'deepeval'", specifier = ">=3.5.0,<5.0.0" }, { name = "httpx", marker = "extra == 'langgraph'", specifier = ">=0.27.0" }, - { name = "jinja2", marker = "extra == 'simulation'", specifier = ">=3.1.0" }, + { name = "jinja2", marker = "extra == 'simulation'", specifier = ">=3.1.6" }, { name = "langchain", marker = "extra == 'langgraph'", specifier = ">=1.0.0" }, { name = "langchain-community", marker = "extra == 'ragas'", specifier = ">=0.3.0,<0.4.2" }, { name = "langchain-mcp-adapters", marker = "extra == 'langgraph'", specifier = ">=0.1.0" },