Shell API and CLI: Add option for staging additional runtime targets

src/buildstream/_frontend/cli.py

@@ -20,6 +20,7 @@
 import shutil
 import click
 from .. import _yaml
+from .._frontend.app import App
 from .._exceptions import BstError, LoadError, AppError, RemoteError
 from .complete import main_bashcomplete, complete_path, CompleteUnhandled
 from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope
@@ -377,8 +378,6 @@ def cli(context, **kwargs):
     user preferences configuration file.
     """
 
-    from .app import App
-
     # Create the App, giving it the main arguments
     context.obj = App.create(dict(kwargs))
     context.call_on_close(context.obj.cleanup)
@@ -686,6 +685,13 @@ def show(app, elements, deps, except_, order, format_):
     metavar="HOSTPATH PATH",
     help="Mount a file or directory into the sandbox",
 )
+@click.option(
+    "--with",
+    "other_targets",
+    type=click.Path(readable=False),
+    multiple=True,
+    help="A additional target to stage into an element's sandbox environment",
+)
 @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox")
 @click.option(
     "--use-buildtree",
@@ -723,8 +729,9 @@ def show(app, elements, deps, except_, order, format_):
 @click.argument("command", type=click.STRING, nargs=-1)
 @click.pass_obj
 def shell(
-    app,
+    app: App,
     target,
+    other_targets,
     command,
     mount,
     isolate,
@@ -748,13 +755,40 @@ def shell(
     otherwise bst may respond to them instead. e.g.
 
     \b
-        bst shell example.bst -- df -h
+        bst shell base.bst -- df -h
 
     Use the --build option to create a temporary sysroot for
     building the element instead.
 
+    Use the --with option to stage the artifacts of other elements
+    into the temporary sysroot to make them available to run e.g.
+
+    \b
+        bst shell --with base.bst example.bst -- cat example.txt
+
     If no COMMAND is specified, the default is to attempt
     to run an interactive shell.
+
+    # Examples:
+
+    \b
+        # Attempt to run an interactive shell with example.bst
+        bst shell example.bst
+        # Attempt to run an df -h with example.bst
+        bst shell example.bst -- df h
+        # In a workspace directory, attempt to shell into the workspace element
+        bst shell
+        # Attempt to run cat from base.bst to read example.txt from example.bst
+        bst shell --with base.bst example.bst -- cat example.txt
+        # Attempt to run an interactive shell with the sources and all dependencies of example.bst
+        bst shell --build example.bst
+
+    For all examples on this page:
+
+    - example.bst is a simple import element with no dependencies
+       that imports a file called example.txt
+    - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc).
+
     """
 
     # Buildtree can only be used with build shells
@@ -764,6 +798,7 @@ def shell(
     scope = _Scope.BUILD if build_ else _Scope.RUN
 
     with app.initialized():
+        assert app.stream, "Must have Stream initialised"
         if not target:
             target = app.stream.get_default_target()
             if not target:
@@ -772,19 +807,35 @@ def shell(
         mounts = [_HostMount(path, host_path) for host_path, path in mount]
 
         try:
-            exitcode = app.stream.shell(
-                target,
-                scope,
-                app.shell_prompt,
-                mounts=mounts,
-                isolate=isolate,
-                command=command,
-                usebuildtree=cli_buildtree,
-                artifact_remotes=artifact_remotes,
-                source_remotes=source_remotes,
-                ignore_project_artifact_remotes=ignore_project_artifact_remotes,
-                ignore_project_source_remotes=ignore_project_source_remotes,
-            )
+            if other_targets:
+                exitcode = app.stream.shell_with(
+                    target,
+                    other_targets,
+                    scope,
+                    app.shell_prompt,
+                    mounts=mounts,
+                    isolate=isolate,
+                    command=command,
+                    usebuildtree=cli_buildtree,
+                    artifact_remotes=artifact_remotes,
+                    source_remotes=source_remotes,
+                    ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+                    ignore_project_source_remotes=ignore_project_source_remotes,
+                )
+            else:
+                exitcode = app.stream.shell(
+                    target,
+                    scope,
+                    app.shell_prompt,
+                    mounts=mounts,
+                    isolate=isolate,
+                    command=command,
+                    usebuildtree=cli_buildtree,
+                    artifact_remotes=artifact_remotes,
+                    source_remotes=source_remotes,
+                    ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+                    ignore_project_source_remotes=ignore_project_source_remotes,
+                )
         except BstError as e:
             raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e
 

src/buildstream/_loader/loader.py

@@ -14,14 +14,19 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
+from buildstream import utils
+import sys
+import tempfile
+from typing import Callable, Generator
 import os
-from contextlib import suppress
+from contextlib import suppress, contextmanager
+from ruamel.yaml import CommentedMap
 
 from .._exceptions import LoadError
 from ..exceptions import LoadErrorReason
 from .. import _yaml
 from ..element import Element
-from ..node import Node
+from ..node import Node, MappingNode
 from .._profile import Topics, PROFILER
 from .._includes import Includes
 from .._utils import valid_chars_name
@@ -73,7 +78,7 @@ def __init__(self, project, *, parent=None, provenance_node=None):
         self._links = {}  # Dict of link target target paths indexed by link element paths
         self._loaders = {}  # Dict of junction loaders
         self._loader_search_provenances = {}  # Dictionary of provenance nodes of ongoing child loader searches
-
+        self._fullpath_overrides: dict[str, str] = {}  # Dictionary: Original Path, Replacement Path
         self._includes = Includes(self, copy_tree=True)
 
         assert project.name is not None
@@ -251,10 +256,58 @@ def foreach_parent(parent):
         for parent in self._alternative_parents:
             yield from foreach_parent(parent)
 
+    # temporary_modified_element()
+    #
+    # Temporarily modify an element by loading the element and applying modify_elements_function to make the modifications
+    #
+    #
+    # Args:
+    #    target (str): The element-path relative bst file
+    #    modify_element_function (Callable[[CommentedMap],None]): A function to modify a given CommentedMap
+    #
+    @contextmanager
+    def temporary_modified_element(
+        self, target: str, modify_element_function: Callable[[CommentedMap], None]
+    ) -> Generator[None, None, None]:
+
+        _, target_name, target_loader = self._parse_name(target, MappingNode.from_dict({}))
+
+        target_path = os.path.join(target_loader._basedir, target_name)
+        target_node: CommentedMap = _yaml.roundtrip_load(target_path)
+
+        modify_element_function(target_node)
+
+        with utils._tempnamedfile(prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" ) as temp_target_file:
+            _yaml.roundtrip_dump(target_node, temp_target_file)
+            temp_target_file.flush()
+            target_loader._set_fullpath_override(target_name, temp_target_file.name)
+
+            try:
+                yield
+            finally:
+                target_loader._set_fullpath_override(target_name, None)
+
     ###########################################
     #            Private Methods              #
     ###########################################
 
+    # _set_fullpath_override()
+    #
+    # Set an fullpath override for a element-path relative bst file
+    #
+    # This enables runtime modified elements to be pulled from a temporary directory
+    # Passing None as a fullpath remove the entry
+    #
+    # Args:
+    #    filename (str): The element-path relative bst file
+    #    fullpath (str|None): A fullpath to the bst file, or None
+    #
+    def _set_fullpath_override(self, filename: str, fullpath: str | None):
+        if fullpath:
+            self._fullpath_overrides[filename] = fullpath
+        else:
+            self._fullpath_overrides.pop(filename, None)
+
     # _load_file_no_deps():
     #
     # Load a bst file as a LoadElement
@@ -275,8 +328,8 @@ def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=Fal
 
         self._assert_element_name(filename, provenance_node)
 
-        # Load the data and process any conditional statements therein
-        fullpath = os.path.join(self._basedir, filename)
+        fullpath = self._fullpath_overrides.get(filename, os.path.join(self._basedir, filename))
+
         try:
             node = _yaml.load(
                 fullpath, shortname=filename, copy_tree=self.load_context.rewritable, project=self.project
@@ -1014,7 +1067,9 @@ def _shallow_load_path(self, path, provenance_node):
     #            - (str): name of the element
     #            - (Loader): loader for sub-project
     #
-    def _parse_name(self, name, provenance_node, *, load_subprojects=True):
+    def _parse_name(
+        self, name: str, provenance_node: MappingNode, *, load_subprojects: bool = True
+    ) -> tuple[str | None, str, "Loader"]:
         # We allow to split only once since deep junctions names are forbidden.
         # Users who want to refer to elements in sub-sub-projects are required
         # to create junctions on the top level project.

src/buildstream/_stream.py

@@ -16,6 +16,7 @@
 #        Jürg Billeter <juerg.billeter@codethink.co.uk>
 #        Tristan Maat <tristan.maat@codethink.co.uk>
 
+
 import itertools
 import os
 import sys
@@ -27,7 +28,10 @@
 from contextlib import contextmanager, suppress
 from collections import deque
 from typing import List, Tuple, Optional, Iterable, Callable
+from ruamel.yaml import CommentedMap
+
 
+from ._context import Context
 from ._artifactelement import verify_artifact_ref, ArtifactElement
 from ._artifactproject import ArtifactProject
 from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
@@ -44,7 +48,7 @@
 )
 from .element import Element
 from ._profile import Topics, PROFILER
-from ._project import ProjectRefStorage
+from ._project import ProjectRefStorage, Project
 from ._remotespec import RemoteSpec
 from ._state import State
 from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount
@@ -79,11 +83,11 @@ def __init__(
         #
         # Private members
         #
-        self._context = context
+        self._context: Context = context
         self._artifacts = None
         self._elementsourcescache = None
         self._sourcecache = None
-        self._project = None
+        self._project: Optional[Project] = None
         self._state = State(session_start)  # Owned by Stream, used by Core to set state
         self._notification_queue = deque()
 
@@ -163,7 +167,7 @@ def load_selection(
         ignore_project_artifact_remotes: bool = False,
         ignore_project_source_remotes: bool = False,
         need_state: bool = True,
-    ):
+    ) -> list[Element]:
         with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)):
             target_objects = self._load(
                 targets,
@@ -235,6 +239,53 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
 
                     task.add_current_progress()
 
+    # shell_with()
+    #
+    # Run a shell with other targets.
+    #
+    # Automatically creates a temporary target based on 'target' with 'other_targets' as runtime or build dependencies.
+    #
+    # Note: Method will build the temporary target, before entering into it's shell.
+    #
+    # Args:
+    #    target (str): The name of the element to run the shell for
+    #    other_targets: (Iterable[str]): The name of the other elements to run the shell with.
+    #    scope: _Scope: Either BUILD or RUN
+    #    *args, **kwargs:  Passed to shell() untouched.
+    #
+    # Returns:
+    #    (int): The exit code of the launched shell
+    #
+    def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, *args, **kwargs):
+
+        assert self._project, "Must have a project"
+        assert self._project.loader, "Project must have loader"
+
+        def add_deps_to_element(target_node: CommentedMap):
+            if scope == _Scope.RUN:
+                r_depends = target_node.get("runtime-depends", [])
+
+                for other_target in other_targets:
+                    r_depends.append(other_target)
+
+                target_node["runtime-depends"] = r_depends
+            elif scope == _Scope.BUILD:
+                r_depends = target_node.get("build-depends", [])
+
+                for other_target in other_targets:
+                    r_depends.append(other_target)
+
+                target_node["build-depends"] = r_depends
+            else:
+                raise StreamError(
+                    "Only BUILD and RUN scopes are supported",
+                    detail="Use the --build and --use-buildtree options to shell into a build tree",
+                    reason="only-build-run-supported",
+                )
+
+        with self._project.loader.temporary_modified_element(target, add_deps_to_element):
+            return self.shell(target, scope, *args, **kwargs)
+
     # shell()
     #
     # Run a shell
@@ -243,6 +294,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source
     #    target: The name of the element to run the shell for
     #    scope: The scope for the shell, only BUILD or RUN are valid (_Scope)
     #    prompt: A function to return the prompt to display in the shell
+    #    other_targets (Iterable[str]): The name of other elements to stage in the shell
     #    unique_id: (str): A unique_id to use to lookup an Element instance
     #    mounts: Additional directories to mount into the sandbox
     #    isolate (bool): Whether to isolate the environment like we do in builds
@@ -1043,6 +1095,7 @@ def workspace_open(
                 self.workspace_close(target._get_full_name(), remove_dir=not no_checkout)
 
             if not custom_dir:
+                assert self._context.workspacedir, "Must have workspace dir"
                 directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name))
                 if directory[-4:] == ".bst":
                     directory = directory[:-4]
@@ -2117,6 +2170,8 @@ def _expand_and_classify_targets(
             # project directory and element path prefix, to produce only element names.
             #
             all_elements = []
+            assert self._project, "Must have a project"
+            assert self._project.element_path, "Must have a project"
             element_path_length = len(self._project.element_path) + 1
             for dirpath, _, filenames in os.walk(self._project.element_path):
                 for filename in filenames:
@@ -2136,6 +2191,7 @@ def _expand_and_classify_targets(
 
         # Glob the artifact names and add the results to the set
         #
+        assert self._artifacts, "Must have artifacts"
         for glob in artifact_globs:
             glob_results = self._artifacts.list_artifacts(glob=glob)
             for artifact_name in glob_results: