From 7a4607f3e589d9521ccd895b0126a35170ac9700 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Fri, 3 Jul 2026 15:27:10 +0100 Subject: [PATCH 1/8] Add app type hint to cli shell function --- src/buildstream/_frontend/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index c3f1b090b..d5ea58256 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -20,6 +20,7 @@ import shutil import click from .. import _yaml +from .._frontend.app import App from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope @@ -723,7 +724,7 @@ def show(app, elements, deps, except_, order, format_): @click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj def shell( - app, + app: App, target, command, mount, From cb880e12eb188a20dcdda2871133dbb2a6ed5adc Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Fri, 3 Jul 2026 15:34:10 +0100 Subject: [PATCH 2/8] Shell API and CLI: Add option for staging additional runtime targets This enables users to add additional functionality such as debug tooling in the shell sandbox, without needing to modify the target element. This is achived through introducing a new option to shell. All existing API and UX is maintained, to not break existing scripts. An alternative design was considered, to have the additional elements as positional arguments similar to the existing element, but this would need manual parsing to handle the cases where `--` is present and not present, splitting based on a `.bst` suffix. This UX could be re-visited in future. Example usage: bst shell --with base.bst example.bst -- cat example.txt Where: - example.bst is a simple import element with no dependencies that imports a file called example.txt - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). Changes: - Introduces `test_with_other_targets` integration test to the shell test suite. - Adds `--with` cli option to the shell subcommand and updates it's documentation. - option can be used multiple times by caller, providing a list of targets. - Extends the shell top level calling interface in Buildstream core to accept a list of other_targets - This is where the targets are loaded into elements and checked to make sure they are present - Extends the shell element implementation to accept a list of other targets - This is where the other elements are staged and integrated into the sandbox --- src/buildstream/_frontend/cli.py | 80 ++++++++++++++++++++++++------- src/buildstream/_loader/loader.py | 25 ++++++++-- src/buildstream/_stream.py | 77 +++++++++++++++++++++++++++-- src/buildstream/_yaml.pyi | 3 ++ src/buildstream/element.py | 5 ++ src/buildstream/types.pyi | 8 ++-- tests/integration/shell.py | 45 ++++++++++++++++- 7 files changed, 213 insertions(+), 30 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index d5ea58256..18b2e328a 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -378,8 +378,6 @@ def cli(context, **kwargs): user preferences configuration file. """ - from .app import App - # Create the App, giving it the main arguments context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) @@ -687,6 +685,13 @@ def show(app, elements, deps, except_, order, format_): metavar="HOSTPATH PATH", help="Mount a file or directory into the sandbox", ) +@click.option( + "--with", + "other_targets", + type=click.Path(readable=False), + multiple=True, + help="A additional target to stage into an element's sandbox environment", +) @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox") @click.option( "--use-buildtree", @@ -726,6 +731,7 @@ def show(app, elements, deps, except_, order, format_): def shell( app: App, target, + other_targets, command, mount, isolate, @@ -749,13 +755,38 @@ def shell( otherwise bst may respond to them instead. e.g. \b - bst shell example.bst -- df -h + bst shell base.bst -- df -h Use the --build option to create a temporary sysroot for building the element instead. + Use the --with option to stage the artifacts of other elements + into the temporary sysroot to make them available to run e.g. + + \b + bst shell --with base.bst example.bst -- cat example.txt + If no COMMAND is specified, the default is to attempt to run an interactive shell. + + # Examples: + + \b + # Attempt to run an interactive shell with example.bst + bst shell example.bst + # Attempt to run an df -h with example.bst + bst shell example.bst -- df h + # In a workspace directory, attempt to shell into the workspace element + bst shell + # Attempt to run cat from base.bst to read example.txt from example.bst + bst shell --with base.bst example.bst -- cat example.txt + # Attempt to run an interactive shell with the sources and all dependencies of example.bst + bst shell --build example.bst + + For all examples on this page: + - example.bst is a simple import element with no dependencies + that imports a file called example.txt + - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). """ # Buildtree can only be used with build shells @@ -765,6 +796,7 @@ def shell( scope = _Scope.BUILD if build_ else _Scope.RUN with app.initialized(): + assert app.stream, "Must have Stream initialised" if not target: target = app.stream.get_default_target() if not target: @@ -773,19 +805,35 @@ def shell( mounts = [_HostMount(path, host_path) for host_path, path in mount] try: - exitcode = app.stream.shell( - target, - scope, - app.shell_prompt, - mounts=mounts, - isolate=isolate, - command=command, - usebuildtree=cli_buildtree, - artifact_remotes=artifact_remotes, - source_remotes=source_remotes, - ignore_project_artifact_remotes=ignore_project_artifact_remotes, - ignore_project_source_remotes=ignore_project_source_remotes, - ) + if other_targets: + exitcode = app.stream.shell_with( + target, + other_targets, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) + else: + exitcode = app.stream.shell( + target, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index af98504cc..64aaf9ec0 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -21,7 +21,7 @@ from ..exceptions import LoadErrorReason from .. import _yaml from ..element import Element -from ..node import Node +from ..node import Node, MappingNode from .._profile import Topics, PROFILER from .._includes import Includes from .._utils import valid_chars_name @@ -73,7 +73,7 @@ def __init__(self, project, *, parent=None, provenance_node=None): self._links = {} # Dict of link target target paths indexed by link element paths self._loaders = {} # Dict of junction loaders self._loader_search_provenances = {} # Dictionary of provenance nodes of ongoing child loader searches - + self._fullpath_overrides: dict[str, str] = {} # Dictionary: Original Path, Replacement Path self._includes = Includes(self, copy_tree=True) assert project.name is not None @@ -255,6 +255,19 @@ def foreach_parent(parent): # Private Methods # ########################################### + # _set_fullpath_overrides() + # + # Set an fullpath override for a element-path relative bst file + # + # This enables runtime modified elements to be pulled from a temporary directory + # + # Args: + # filename (str): The element-path relative bst file + # fullpath (str): A fullpath to the bst file + # + def _set_fullpath_overrides(self, filename: str, fullpath: str): + self._fullpath_overrides[filename] = fullpath + # _load_file_no_deps(): # # Load a bst file as a LoadElement @@ -275,8 +288,8 @@ def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=Fal self._assert_element_name(filename, provenance_node) - # Load the data and process any conditional statements therein - fullpath = os.path.join(self._basedir, filename) + fullpath = self._fullpath_overrides.get(filename, os.path.join(self._basedir, filename)) + try: node = _yaml.load( fullpath, shortname=filename, copy_tree=self.load_context.rewritable, project=self.project @@ -1014,7 +1027,9 @@ def _shallow_load_path(self, path, provenance_node): # - (str): name of the element # - (Loader): loader for sub-project # - def _parse_name(self, name, provenance_node, *, load_subprojects=True): + def _parse_name( + self, name: str, provenance_node: MappingNode, *, load_subprojects: bool = True + ) -> tuple[str | None, str, "Loader"]: # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index a1d88d667..9dde28d66 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -16,6 +16,7 @@ # Jürg Billeter # Tristan Maat + import itertools import os import sys @@ -27,7 +28,11 @@ from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable +from ruamel.yaml import CommentedMap + +from ._context import Context +from .node import MappingNode from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError @@ -44,7 +49,7 @@ ) from .element import Element from ._profile import Topics, PROFILER -from ._project import ProjectRefStorage +from ._project import ProjectRefStorage, Project from ._remotespec import RemoteSpec from ._state import State from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount @@ -79,11 +84,11 @@ def __init__( # # Private members # - self._context = context + self._context: Context = context self._artifacts = None self._elementsourcescache = None self._sourcecache = None - self._project = None + self._project: Optional[Project] = None self._state = State(session_start) # Owned by Stream, used by Core to set state self._notification_queue = deque() @@ -163,7 +168,7 @@ def load_selection( ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, need_state: bool = True, - ): + ) -> list[Element]: with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): target_objects = self._load( targets, @@ -235,6 +240,65 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source task.add_current_progress() + # shell_with() + # + # Run a shell with other targets. + # + # Automatically creates a temporary target based on 'target' with 'other_targets' as runtime or build dependencies. + # + # Note: Method will build the temporary target, before entering into it's shell. + # + # Args: + # target (str): The name of the element to run the shell for + # other_targets: (Iterable[str]): The name of the other elements to run the shell with. + # scope: _Scope: Either BUILD or RUN + # *args, **kwargs: Passed to shell() untouched. + # + # Returns: + # (int): The exit code of the launched shell + # + def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, *args, **kwargs): + + assert self._project, "Must have a project" + assert self._project.loader, "Project must have loader" + + _, target_name, target_loader = self._project.loader._parse_name(target, MappingNode.from_dict({})) + + target_path = os.path.join(target_loader._basedir, target_name) + target_node: CommentedMap = _yaml.roundtrip_load(target_path) + + if scope == _Scope.RUN: + r_depends = target_node.get("runtime-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["runtime-depends"] = r_depends + elif scope == _Scope.BUILD: + r_depends = target_node.get("build-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["build-depends"] = r_depends + else: + raise StreamError( + "Only BUILD and RUN scopes are supported", + detail="Use the --build and --use-buildtree options to shell into a build tree", + reason="only-build-run-supported", + ) + + with tempfile.NamedTemporaryFile( + delete_on_close=False, prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" + ) as temp_target_file: + _yaml.roundtrip_dump(target_node, temp_target_file) + temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. + + target_loader._set_fullpath_overrides(target_name, temp_target_file.name) + + self.build([target]) + return self.shell(target, scope, *args, **kwargs) + # shell() # # Run a shell @@ -243,6 +307,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source # target: The name of the element to run the shell for # scope: The scope for the shell, only BUILD or RUN are valid (_Scope) # prompt: A function to return the prompt to display in the shell + # other_targets (Iterable[str]): The name of other elements to stage in the shell # unique_id: (str): A unique_id to use to lookup an Element instance # mounts: Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds @@ -1043,6 +1108,7 @@ def workspace_open( self.workspace_close(target._get_full_name(), remove_dir=not no_checkout) if not custom_dir: + assert self._context.workspacedir, "Must have workspace dir" directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name)) if directory[-4:] == ".bst": directory = directory[:-4] @@ -2117,6 +2183,8 @@ def _expand_and_classify_targets( # project directory and element path prefix, to produce only element names. # all_elements = [] + assert self._project, "Must have a project" + assert self._project.element_path, "Must have a project" element_path_length = len(self._project.element_path) + 1 for dirpath, _, filenames in os.walk(self._project.element_path): for filename in filenames: @@ -2136,6 +2204,7 @@ def _expand_and_classify_targets( # Glob the artifact names and add the results to the set # + assert self._artifacts, "Must have artifacts" for glob in artifact_globs: glob_results = self._artifacts.list_artifacts(glob=glob) for artifact_name in glob_results: diff --git a/src/buildstream/_yaml.pyi b/src/buildstream/_yaml.pyi index 224abc51a..b301cab68 100644 --- a/src/buildstream/_yaml.pyi +++ b/src/buildstream/_yaml.pyi @@ -12,7 +12,10 @@ # limitations under the License. # from typing import Optional +from ruamel.yaml import CommentedMap from .node import MappingNode def load(filename: str, shortname: str, copy_tree: bool = False, project: Optional[object] = None) -> MappingNode: ... +def roundtrip_load(filename: str, *, allow_missing: bool = False) -> CommentedMap: ... +def roundtrip_dump(contents, file): ... diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 621568e63..a0b38ef94 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -62,6 +62,9 @@ --------------- """ +# For 3.7+ support, not necessary and deprecated in 3.14+ +from __future__ import annotations + import os import re import stat @@ -74,6 +77,7 @@ from threading import Lock from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence + from pyroaring import BitMap # pylint: disable=no-name-in-module from . import _yaml @@ -2058,6 +2062,7 @@ def _push(self): # prompt (str): A suitable prompt string for PS1 # command (list): An argv to launch in the sandbox # usebuildtree (bool): Use the buildtree as its source + # other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox # # Returns: Exit code def _shell( diff --git a/src/buildstream/types.pyi b/src/buildstream/types.pyi index 8a75b9f13..db9d59d06 100644 --- a/src/buildstream/types.pyi +++ b/src/buildstream/types.pyi @@ -116,10 +116,10 @@ class OverlapAction(Enum): IGNORE: str class _Scope(Enum): - ALL: int - BUILD: int - RUN: int - NONE: int + ALL = 1 + BUILD = 2 + RUN = 3 + NONE = 4 class _KeyStrength(Enum): STRONG: int diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 63c7af8b3..b8d24c424 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -16,12 +16,15 @@ # pylint: disable=redefined-outer-name import os +from typing import Dict, List, Tuple import uuid + import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import +from buildstream._testing.runcli import CliIntegration from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN from buildstream.exceptions import ErrorDomain from buildstream import utils @@ -46,11 +49,26 @@ # mount (tuple): A (host, target) tuple for the `--mount` option # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` +# other_elements (list(str)): Other elements to stage in the sandbox # -def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): +def execute_shell( + cli: CliIntegration, + project: str, + command: List[str], + *, + config: None | Dict = None, + mount: Tuple[str, str] | None = None, + element: str = "base.bst", + isolate: bool = False, + other_elements: List[str] | None = None, +): # Ensure the element is built result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 + if other_elements is not None: + for other_element in other_elements: + result = cli.run_project_config(project=project, project_config=config, args=["build", other_element]) + assert result.exit_code == 0 args = ["shell"] if isolate: @@ -58,6 +76,9 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element="ba if mount is not None: host_path, target_path = mount args += ["--mount", host_path, target_path] + if other_elements is not None: + for other_element in other_elements: + args += ["--with", other_element] args += [element, "--", *command] return cli.run_project_config(project=project, project_config=config, args=args) @@ -86,6 +107,28 @@ def test_executable(cli, datafiles): assert result.output == "Horseys!\n" +# Test staging and running additional targets in the shell of the main target for debugging. +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_with_other_targets(cli, datafiles): + project = str(datafiles) + + # Show we can't cat in a shell for manual/import-file.bst + result = execute_shell(cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst") + assert ( + result.exit_code == -1 + ), "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies" + + # Show we can now cat with base.bst in a shell for manual/import-file.bst + result = execute_shell( + cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst", other_elements=["base.bst"] + ) + assert ( + result.exit_code == 0 + ), "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat" + assert result.output == "This is a test\n" + + # Test shell environment variable explicit assignments @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR) From d6928c64151bf77e51242711fc35d3f094cec906 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 15 Sep 2026 12:26:23 +0100 Subject: [PATCH 3/8] Make set_fullpath_override public --- src/buildstream/_loader/loader.py | 4 ++-- src/buildstream/_stream.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 64aaf9ec0..239bc7f3a 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -255,7 +255,7 @@ def foreach_parent(parent): # Private Methods # ########################################### - # _set_fullpath_overrides() + # set_fullpath_override() # # Set an fullpath override for a element-path relative bst file # @@ -265,7 +265,7 @@ def foreach_parent(parent): # filename (str): The element-path relative bst file # fullpath (str): A fullpath to the bst file # - def _set_fullpath_overrides(self, filename: str, fullpath: str): + def set_fullpath_override(self, filename: str, fullpath: str): self._fullpath_overrides[filename] = fullpath # _load_file_no_deps(): diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 9dde28d66..bafb522bf 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -294,7 +294,7 @@ def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, * _yaml.roundtrip_dump(target_node, temp_target_file) temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. - target_loader._set_fullpath_overrides(target_name, temp_target_file.name) + target_loader.set_fullpath_override(target_name, temp_target_file.name) self.build([target]) return self.shell(target, scope, *args, **kwargs) From 5fd3e5b685baf284e0498b2280671e59703d7568 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 15 Sep 2026 12:52:22 +0100 Subject: [PATCH 4/8] Use loader context manager --- src/buildstream/_loader/loader.py | 47 +++++++++++++++++++++++++++---- src/buildstream/_stream.py | 45 +++++++++++------------------ 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 239bc7f3a..7f141ba27 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -14,8 +14,11 @@ # Authors: # Tristan Van Berkom +import tempfile +from typing import Callable, Generator import os -from contextlib import suppress +from contextlib import suppress, contextmanager +from ruamel.yaml import CommentedMap from .._exceptions import LoadError from ..exceptions import LoadErrorReason @@ -251,22 +254,56 @@ def foreach_parent(parent): for parent in self._alternative_parents: yield from foreach_parent(parent) + # temporary_modified_element() + # + # Temporarily modify an element by loading the element and applying modify_elements_function to make the modifications + # + # + # Args: + # target (str): The element-path relative bst file + # modify_element_function (Callable[[CommentedMap],None]): A function to modify a given CommentedMap + # + @contextmanager + def temporary_modified_element(self,target:str,modify_element_function:Callable[[CommentedMap],None]) -> Generator[None, None, None]: + + _, target_name, target_loader = self._parse_name(target, MappingNode.from_dict({})) + + target_path = os.path.join(target_loader._basedir, target_name) + target_node: CommentedMap = _yaml.roundtrip_load(target_path) + + modify_element_function(target_node) + + with tempfile.NamedTemporaryFile( + delete_on_close=False, prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" + ) as temp_target_file: + _yaml.roundtrip_dump(target_node, temp_target_file) + temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. + target_loader._set_fullpath_override(target_name, temp_target_file.name) + + yield + + target_loader._set_fullpath_override(target_name,None) + ########################################### # Private Methods # ########################################### - # set_fullpath_override() + # _set_fullpath_override() # # Set an fullpath override for a element-path relative bst file # # This enables runtime modified elements to be pulled from a temporary directory + # Passing None as a fullpath remove the entry # # Args: # filename (str): The element-path relative bst file - # fullpath (str): A fullpath to the bst file + # fullpath (str|None): A fullpath to the bst file, or None # - def set_fullpath_override(self, filename: str, fullpath: str): - self._fullpath_overrides[filename] = fullpath + def _set_fullpath_override(self, filename: str, fullpath: str | None): + if fullpath: + self._fullpath_overrides[filename] = fullpath + else: + self._fullpath_overrides.pop(filename, None) # _load_file_no_deps(): # diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index bafb522bf..f78bb553b 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -262,40 +262,29 @@ def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, * assert self._project, "Must have a project" assert self._project.loader, "Project must have loader" - _, target_name, target_loader = self._project.loader._parse_name(target, MappingNode.from_dict({})) + def add_deps_to_element(target_node: CommentedMap): + if scope == _Scope.RUN: + r_depends = target_node.get("runtime-depends", []) - target_path = os.path.join(target_loader._basedir, target_name) - target_node: CommentedMap = _yaml.roundtrip_load(target_path) + for other_target in other_targets: + r_depends.append(other_target) - if scope == _Scope.RUN: - r_depends = target_node.get("runtime-depends", []) - - for other_target in other_targets: - r_depends.append(other_target) - - target_node["runtime-depends"] = r_depends - elif scope == _Scope.BUILD: - r_depends = target_node.get("build-depends", []) + target_node["runtime-depends"] = r_depends + elif scope == _Scope.BUILD: + r_depends = target_node.get("build-depends", []) - for other_target in other_targets: - r_depends.append(other_target) + for other_target in other_targets: + r_depends.append(other_target) - target_node["build-depends"] = r_depends - else: - raise StreamError( - "Only BUILD and RUN scopes are supported", - detail="Use the --build and --use-buildtree options to shell into a build tree", - reason="only-build-run-supported", + target_node["build-depends"] = r_depends + else: + raise StreamError( + "Only BUILD and RUN scopes are supported", + detail="Use the --build and --use-buildtree options to shell into a build tree", + reason="only-build-run-supported", ) - with tempfile.NamedTemporaryFile( - delete_on_close=False, prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" - ) as temp_target_file: - _yaml.roundtrip_dump(target_node, temp_target_file) - temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. - - target_loader.set_fullpath_override(target_name, temp_target_file.name) - + with self._project.loader.temporary_modified_element(target,add_deps_to_element): self.build([target]) return self.shell(target, scope, *args, **kwargs) From 08083da34824e74158dabeeea7f9e3b33d7f931d Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 15 Sep 2026 14:40:55 +0100 Subject: [PATCH 5/8] NamedTemporaryFile delete_on_close for python 3.10 and 3.11 --- src/buildstream/_loader/loader.py | 42 +++++++++++++++++++++++++++---- src/buildstream/_stream.py | 5 ++-- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 7f141ba27..372b765d3 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -14,6 +14,7 @@ # Authors: # Tristan Van Berkom +import sys import tempfile from typing import Callable, Generator import os @@ -264,7 +265,9 @@ def foreach_parent(parent): # modify_element_function (Callable[[CommentedMap],None]): A function to modify a given CommentedMap # @contextmanager - def temporary_modified_element(self,target:str,modify_element_function:Callable[[CommentedMap],None]) -> Generator[None, None, None]: + def temporary_modified_element( + self, target: str, modify_element_function: Callable[[CommentedMap], None] + ) -> Generator[None, None, None]: _, target_name, target_loader = self._parse_name(target, MappingNode.from_dict({})) @@ -273,16 +276,18 @@ def temporary_modified_element(self,target:str,modify_element_function:Callable[ modify_element_function(target_node) - with tempfile.NamedTemporaryFile( + # FIXME When 3.12 hits EOL, replace this with tempfile.NamedTemporaryFile itself. + with _legacy_named_temporary_file_delete_on_close( delete_on_close=False, prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" ) as temp_target_file: _yaml.roundtrip_dump(target_node, temp_target_file) temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. target_loader._set_fullpath_override(target_name, temp_target_file.name) - yield - - target_loader._set_fullpath_override(target_name,None) + try: + yield + finally: + target_loader._set_fullpath_override(target_name, None) ########################################### # Private Methods # @@ -1141,3 +1146,30 @@ def _clean_caches(self): self._meta_elements = {} self._elements = {} + + +# _legacy_named_temporary_file_delete_on_close() +# +# Helper for python 3.10 and 3.11 support +# +# NamedTemporaryFile attribute `delete_on_close=False` was not added until 3.12 +# +# FIXME: When 3.11 hits end of life remove this function. +# +@contextmanager +def _legacy_named_temporary_file_delete_on_close(delete_on_close=False, prefix=None, suffix=None): + + assert not delete_on_close, "Don't use this function unless you explicitly need delete_on_close set to false" + if sys.version_info >= (3, 12): + # Use the `delete_on_close` attribute if it's available + yield tempfile.NamedTemporaryFile(delete_on_close=delete_on_close, prefix=prefix, suffix=suffix) + else: + # Otherwise implement it for ourselves. + file = tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, delete=False) + try: + yield file + finally: + try: + os.unlink(file.name) + except OSError: + pass diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index f78bb553b..dbabc1d27 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -32,7 +32,6 @@ from ._context import Context -from .node import MappingNode from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError @@ -282,9 +281,9 @@ def add_deps_to_element(target_node: CommentedMap): "Only BUILD and RUN scopes are supported", detail="Use the --build and --use-buildtree options to shell into a build tree", reason="only-build-run-supported", - ) + ) - with self._project.loader.temporary_modified_element(target,add_deps_to_element): + with self._project.loader.temporary_modified_element(target, add_deps_to_element): self.build([target]) return self.shell(target, scope, *args, **kwargs) From c481b771dfcf64c17ce1a6455dc8911fd30e89e1 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 15 Sep 2026 14:48:04 +0100 Subject: [PATCH 6/8] Fix doc failure --- src/buildstream/_frontend/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 18b2e328a..e6c895ede 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -784,9 +784,11 @@ def shell( bst shell --build example.bst For all examples on this page: + - example.bst is a simple import element with no dependencies that imports a file called example.txt - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). + """ # Buildtree can only be used with build shells From f4b45ac5851cd28c2e6deda9782a0330680dd3bd Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 15 Sep 2026 17:06:09 +0100 Subject: [PATCH 7/8] WIP: FIXUP: Don't build as part of shell Needs more testing --- src/buildstream/_stream.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index dbabc1d27..7bc34c2f4 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -284,7 +284,6 @@ def add_deps_to_element(target_node: CommentedMap): ) with self._project.loader.temporary_modified_element(target, add_deps_to_element): - self.build([target]) return self.shell(target, scope, *args, **kwargs) # shell() From c5a75694e34c841544c00ebbf7f8f28e98c8ef0e Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 29 Sep 2026 10:41:16 +0100 Subject: [PATCH 8/8] Use utils._tempnamedfile --- src/buildstream/_loader/loader.py | 35 +++---------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 372b765d3..df454ec98 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -14,6 +14,7 @@ # Authors: # Tristan Van Berkom +from buildstream import utils import sys import tempfile from typing import Callable, Generator @@ -276,12 +277,9 @@ def temporary_modified_element( modify_element_function(target_node) - # FIXME When 3.12 hits EOL, replace this with tempfile.NamedTemporaryFile itself. - with _legacy_named_temporary_file_delete_on_close( - delete_on_close=False, prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" - ) as temp_target_file: + with utils._tempnamedfile(prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" ) as temp_target_file: _yaml.roundtrip_dump(target_node, temp_target_file) - temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. + temp_target_file.flush() target_loader._set_fullpath_override(target_name, temp_target_file.name) try: @@ -1146,30 +1144,3 @@ def _clean_caches(self): self._meta_elements = {} self._elements = {} - - -# _legacy_named_temporary_file_delete_on_close() -# -# Helper for python 3.10 and 3.11 support -# -# NamedTemporaryFile attribute `delete_on_close=False` was not added until 3.12 -# -# FIXME: When 3.11 hits end of life remove this function. -# -@contextmanager -def _legacy_named_temporary_file_delete_on_close(delete_on_close=False, prefix=None, suffix=None): - - assert not delete_on_close, "Don't use this function unless you explicitly need delete_on_close set to false" - if sys.version_info >= (3, 12): - # Use the `delete_on_close` attribute if it's available - yield tempfile.NamedTemporaryFile(delete_on_close=delete_on_close, prefix=prefix, suffix=suffix) - else: - # Otherwise implement it for ourselves. - file = tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, delete=False) - try: - yield file - finally: - try: - os.unlink(file.name) - except OSError: - pass